Merge pull request #44 from schubergphilis/add-custom-patch-script

Handle XenServer patching automatically

.gitignore

@@ -2,4 +2,5 @@
 *.swp
 config
 *.iml
-.idea/
+.idea/
+xenserver_patches/

cloudstackops/xenserver.py

@@ -24,6 +24,7 @@
 import sys
 import time
 import os
+import requests
 
 # Fabric
 from fabric.api import *
@@ -49,9 +50,12 @@
 # Class to handle XenServer patching
 class xenserver():
 
-    def __init__(self, ssh_user='root', threads=5):
+    def __init__(self, ssh_user='root', threads=5, pre_empty_script='xenserver_pre_empty_script.sh',
+                 post_empty_script='xenserver_post_empty_script.sh'):
         self.ssh_user = ssh_user
         self.threads = threads
+        self.pre_empty_script = pre_empty_script
+        self.post_empty_script = post_empty_script
 
     # Wait for hypervisor to become alive again
     def check_connect(self, host):
@@ -74,16 +78,15 @@ def check_connect(self, host):
         # Remove progress indication
         sys.stdout.write("\033[F")
         print "Note: Host " + host.name + " is able to do XE stuff again!                                  "
-        print "Note: Waiting 30s to allow the hypervisor to connect.."
-        time.sleep(30)
+        print "Note: Waiting 60s to allow the hypervisor to connect.."
+        time.sleep(60)
         return True
 
     # Check if we can use xapi
     def check_xapi(self, host):
         try:
-            with settings(host_string=self.ssh_user + "@" + host.ipaddress):
-                with warn_only():
-                    result = fab.run("xe host-enable host=" + host.name)
+            with settings(warn_only=True, host_string=self.ssh_user + "@" + host.ipaddress):
+                result = fab.run("xe host-enable host=" + host.name)
                 if result.return_code == 0:
                     return True
                 else:
@@ -169,11 +172,16 @@ def host_evacuate(self, host):
 
     # Reboot a host when all conditions are met
     def host_reboot(self, host, halt_hypervisor=False):
-        # Disbale host
+        # Disable host
         if self.host_disable(host) is False:
             print "Error: Disabling host " + host.name + " failed."
             return False
 
+        # Execute pre-empty-script
+        if self.exec_script_on_hypervisor(host, self.pre_empty_script) is False:
+            print "Error: Executing script '" + self.pre_empty_script + "' on host " + host.name + " failed."
+            return False
+
         # Then evacuate it
         if self.host_evacuate(host) is False:
             print "Error: Evacuating host " + host.name + " failed."
@@ -185,6 +193,11 @@ def host_reboot(self, host, halt_hypervisor=False):
             return False
         print "Note: Host " + host.name + " has no VMs running, continuing"
 
+        # Execute post-empty-script
+        if self.exec_script_on_hypervisor(host, self.post_empty_script) is False:
+            print "Error: Executing script '" + self.post_empty_script + "' on host " + host.name + " failed."
+            return False
+
         # Finally reboot it
         try:
             with settings(host_string=self.ssh_user + "@" + host.ipaddress):
@@ -209,6 +222,16 @@ def host_reboot(self, host, halt_hypervisor=False):
             print "Error: Enabling host " + host.name + " failed."
             return False
 
+    # Execute script on hypervisor
+    def exec_script_on_hypervisor(self, host, script):
+        script = script.split('/')[-1]
+        print "Note: Executing script '%s' on host %s.." % (script, host.name)
+        try:
+            with settings(show('output'), host_string=self.ssh_user + "@" + host.ipaddress):
+                return fab.run("bash /tmp/" + script)
+        except:
+            return False
+
     # Get VM count of a hypervisor
     def host_get_vms(self, host):
         try:
@@ -275,6 +298,12 @@ def put_scripts(self, host):
                     '/tmp/xenserver_fake_pvtools.sh', mode=0755)
                 put('xenserver_parallel_evacuate.py',
                     '/tmp/xenserver_parallel_evacuate.py', mode=0755)
+                if len(self.pre_empty_script) > 0:
+                    put(self.pre_empty_script,
+                        '/tmp/' + self.pre_empty_script.split('/')[-1], mode=0755)
+                if len(self.post_empty_script) > 0:
+                    put(self.post_empty_script,
+                        '/tmp/' + self.post_empty_script.split('/')[-1], mode=0755)
             return True
         except:
             print "Warning: Could not upload check scripts to host " + host.name + ". Continuing anyway."
@@ -308,3 +337,65 @@ def get_bond_status(self, host):
                 return fab.run("python /tmp/xenserver_check_bonds.py | awk {'print $1'} | tr -d \":\"")
         except:
             return False
+
+    # Download XenServer patch
+    def download_patch(self, url):
+        filename = url.split("/")[-1]
+
+        directory = "xenserver_patches"
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+
+        destination_file = os.getcwd() + '/' + directory + '/' + filename
+        try:
+            local_length = int(os.path.getsize(destination_file))
+        except:
+            local_length = 0
+
+        print "Note: Executing request.."
+        try:
+            response = requests.get(url, stream=True)
+            remote_length = int(response.headers.get('Content-Length', 0))
+            if not response.ok:
+                return False
+        except:
+            return False
+
+        # Do we need to download?
+        print "Note: The remote length is %s, local length is %s" % (remote_length, local_length)
+
+        if remote_length == local_length:
+            print "Note: Skipping download because file is already downloaded."
+            return True
+
+        with open(destination_file, 'wb') as handle:
+            # Download file
+            print "Note: Downloading file.."
+
+            for block in response.iter_content(1024):
+                handle.write(block)
+            return True
+
+    # Upload patches to poolmaster
+    def put_patches_to_poolmaster(self, host):
+        print "Note: Uploading patches to poolmaster.."
+        try:
+            with settings(host_string=self.ssh_user + "@" + host.ipaddress):
+                run('rm -rf /root/xenserver_patches/')
+                run('mkdir -p /root/xenserver_patches')
+                put('xenserver_patches/*', '/root/xenserver_patches')
+                put('xenserver_upload_patches_to_poolmaster.sh',
+                    '/root/xenserver_patches/xenserver_upload_patches_to_poolmaster.sh', mode=0755)
+            return True
+        except:
+            print "Warning: Could not upload patches to host " + host.name + "."
+            return False
+
+    # Upload patches to XenServer
+    def upload_patches_to_xenserver(self, host):
+        print "Note: We're uploading the patches to XenServer"
+        try:
+            with settings(show('output'), host_string=self.ssh_user + "@" + host.ipaddress):
+                return fab.run("bash /root/xenserver_patches/xenserver_upload_patches_to_poolmaster.sh")
+        except:
+            return False

xenserver_post_empty_script.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+echo "This is the post_empty script you could customise."
+
+#echo "Downgrading openvswitch RPM to the XenServer default"
+rpm -Uvh http://10.200.10.10/software/xenserver/openvswitch-1.4.6-143.9926.i386.rpm --force --nodeps
+
+echo "Applying patches"
+HOST_UUID=$(xe host-list name-label=${HOSTNAME} --minimal)
+
+# Check for 6.5
+cat /etc/redhat-release | grep "6.5"
+if [ $? -eq 0 ]; then
+    SERVICE_PACK_PATCH=XS65ESP1
+fi
+# Check for 6.2
+cat /etc/redhat-release | grep "6.2"
+if [ $? -eq 0 ]; then
+    SERVICE_PACK_PATCH=XS62ESP1
+fi
+
+# First apply SP1
+xe patch-list name-label=${SERVICE_PACK_PATCH} params=hosts --minimal | tr ',' '\n' | grep ${HOST_UUID}
+if [ $? -eq 0 ]; then
+   echo Service Pack ${SERVICE_PACK_PATCH} is already installed, skipping.
+else
+   echo Installing ${SERVICE_PACK_PATCH}...
+   PATCH_UUID=$(xe patch-list name-label=${SERVICE_PACK_PATCH} | grep uuid | sed -e 's/^.*: //g')
+   if [ ${PATCH_UUID} ]; then
+       xe patch-apply uuid=${PATCH_UUID} host-uuid=${HOST_UUID}
+   fi
+fi
+
+# Apply any other available patch
+XEN_ALL_PATCHES=$(xe patch-list params=name-label --minimal | tr ',' '\n' )
+XEN_INSTALLED_PATCHES=$(xe patch-list hosts:contains=${HOST_UUID} params=name-label --minimal | tr ',' '\n' )
+
+for patch in ${XEN_ALL_PATCHES}; do
+    echo "Checking patch " ${patch}
+
+    # Check if already included
+    echo ${XEN_INSTALLED_PATCHES} | grep ${patch} 2>&1 >/dev/null
+    if [ $? -eq 0 ]; then
+       echo Patch ${patch} is already installed, skipping.
+    else
+       echo Installing $patch...
+       PATCH_UUID=$(xe patch-list name-label=${patch}| grep uuid | sed -e 's/^.*: //g')
+       if [ ${PATCH_UUID} ]; then
+           xe patch-apply uuid=${PATCH_UUID} host-uuid=${HOST_UUID}
+       fi
+    fi
+done
+
+echo "Upgrading drivers"
+yum -y install bnx2x-* fnic* qla2* glnic* qlge* tg3* openvswitch-modules-xen*
+if [ $? -eq 0 ]; then
+    echo "Yum commnand returned non-zero"
+    exit 1
+fi
+
+SYSTEM_HARDWARE=$(dmidecode -s system-product-name | grep -v "#")
+if [[ "${SYSTEM_HARDWARE}" == "ProLiant DL380 G7" ]]; then
+    echo "Skip HPSA on HP ProLiant DL380 G7 or else the box won't boot"
+else
+    yum -y install hpsa*
+    if [ $? -eq 0 ]; then
+    echo "Yum commnand returned non-zero"
+    exit 1
+fi
+fi
+yum -y upgrade nicira-ovs-hypervisor-node
+if [ $? -eq 0 ]; then
+    echo "Yum commnand returned non-zero"
+    exit 1
+fi

xenserver_pre_empty_script.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "This is the pre_empty script you could customise."

xenserver_rolling_reboot.py

@@ -28,6 +28,7 @@
 import time
 import os
 import getopt
+import glob
 from cloudstackops import cloudstackops
 from cloudstackops import xenserver
 # Fabric
@@ -57,6 +58,14 @@ def handleArguments(argv):
     threads = 5
     global halt_hypervisor
     halt_hypervisor = False
+    global pre_empty_script
+    pre_empty_script = 'xenserver_pre_empty_script.sh'
+    global post_empty_script
+    post_empty_script = 'xenserver_post_empty_script.sh'
+    global patch_list_file
+    patch_list_file = 'xenserver_patches_to_install.txt'
+    global preserve_downloads
+    preserve_downloads = False
 
     # Usage message
     help = "Usage: ./" + os.path.basename(__file__) + ' [options]' + \
@@ -68,14 +77,21 @@ def handleArguments(argv):
         '\n  --threads <nr>\t\t\t\tUse this number or concurrent migration threads ' + \
         '\n  --halt\t\t\t\t\tInstead of the default reboot, halt the hypervisor (useful in case of hardware ' \
         'upgrades) ' + \
+        '\n  --pre-empty-script\t\t\t\tBash script to run on hypervisor before starting the live migrations to empty ' \
+        'hypervisor (expected in same folder as this script)' + \
+        '\n  --post-empty-script\t\t\t\tBash script to run on hypervisor after a hypervisor has no more VMs running' \
+        '\n  --patch-list-file\t\t\t\tText file with URLs of patches to download and install. One per line. ' \
+        '(expected in same folder as this script)' + \
+        '\n  --preserve-downloads\t\t\t\tPreserve downloads instead of wiping them and downloading again.' + \
         '\n  --debug\t\t\t\t\tEnable debug mode' + \
         '\n  --exec\t\t\t\t\tExecute for real' + \
         '\n  --prepare\t\t\t\t\tExecute some prepare commands'
 
     try:
         opts, args = getopt.getopt(
             argv, "hc:n:t:p", [
-                "credentials-file=", "clustername=", "ignore-hosts=", "threads=", "halt", "debug", "exec", "prepare"])
+                "credentials-file=", "clustername=", "ignore-hosts=", "threads=", "pre-empty-script=",
+                "post-empty-script=", "patch-list-file=", "preserve-downloads", "halt", "debug", "exec", "prepare"])
     except getopt.GetoptError as e:
         print "Error: " + str(e)
         print help
@@ -95,6 +111,14 @@ def handleArguments(argv):
             ignoreHostList = arg
         elif opt in ("--halt"):
             halt_hypervisor = True
+        elif opt in ("--pre-empty-script"):
+            pre_empty_script = arg
+        elif opt in ("--post-empty-script"):
+            post_empty_script = arg
+        elif opt in ("--patch-list-file"):
+            patch_list_file = arg
+        elif opt in ("--preserve-downloads"):
+            preserve_downloads = True
         elif opt in ("--debug"):
             DEBUG = 1
         elif opt in ("--exec"):
@@ -108,7 +132,7 @@ def handleArguments(argv):
 
     # Ignore host list
     if len(ignoreHostList) > 0:
-        ignoreHosts = ignoreHostList.split(", ")
+        ignoreHosts = ignoreHostList.replace(' ', '').split(",")
     else:
         ignoreHosts = []
 
@@ -125,7 +149,7 @@ def handleArguments(argv):
 c = cloudstackops.CloudStackOps(DEBUG, DRYRUN)
 
 # Init XenServer class
-x = xenserver.xenserver('root', threads)
+x = xenserver.xenserver('root', threads, pre_empty_script, post_empty_script)
 c.xenserver = x
 
 # make credentials file known to our class
@@ -200,7 +224,11 @@ def handleArguments(argv):
     print "  - Turn OFF XenServer poolHA for " + clustername
     print "  - For any hypervisor it will do this (poolmaster " + poolmaster.name + " first):"
     print "      - put it to Disabled aka Maintenance in XenServer"
+    print "      - download the patches in file --patch-list-file '" + patch_list_file + "'"
+    print "         (preserve downloads is set to " + str(preserve_downloads) + ")"
+    print "      - execute the --pre-empty-script script '" + pre_empty_script + "' on the hypervisor"
     print "      - live migrate all VMs off of it using XenServer evacuate command"
+    print "      - execute the --post-empty-script script '" + post_empty_script + "' on the hypervisor"
     print "      - when empty, it will reboot the hypervisor (halting is " + str(halt_hypervisor) + ")"
     print "      - will wait for it to come back online (checks SSH connection)"
     print "      - set the hypervisor to Enabled in XenServer"
@@ -249,6 +277,26 @@ def handleArguments(argv):
         disconnect_all()
         sys.exit(1)
 
+    # Download all XenServer patches
+    if not preserve_downloads:
+        print "Note: Deleting previously downloaded patches"
+        files = glob.glob('xenserver_patches/*.zip')
+        for f in files:
+            print "Note: Removing previously downloaded patch " + f
+            os.remove(f)
+
+    print "Note: Reading patches list '%s'" % patch_list_file
+    with open(patch_list_file) as file_pointer:
+        patches = file_pointer.read().splitlines()
+
+    for patch_url in patches:
+        print "Note: Processing patch '%s'" % patch_url
+        x.download_patch(patch_url)
+
+    # Upload the patches to poolmaster, then to XenServer
+    x.put_patches_to_poolmaster(poolmaster)
+    x.upload_patches_to_xenserver(poolmaster)
+
     # Migrate all VMs off of pool master
     vm_count = x.host_get_vms(poolmaster)
     if vm_count:
@@ -274,8 +322,8 @@ def handleArguments(argv):
         disconnect_all()
         sys.exit(1)
 
-    print "Note: Waiting 30s to allow all hosts connect.."
-    time.sleep(30)
+    print "Note: Waiting 60s to allow all hosts connect.."
+    time.sleep(60)
 
 else:
         print "Warning: Skipping " + poolmaster.name + " due to --ignore-hosts setting"