From b214496dbe51051dcc6fba0ac463d457eff2d926 Mon Sep 17 00:00:00 2001
From: Remi Bergsma <github@remi.nl>
Date: Wed, 3 Aug 2016 16:47:43 +0200
Subject: [PATCH 01/11] Ability to run script before/after live migrations

---
 cloudstackops/xenserver.py     | 32 ++++++++++++++++++++++++++++++--
 xenserver_post_empty_script.sh |  3 +++
 xenserver_pre_empty_script.sh  |  3 +++
 xenserver_rolling_reboot.py    | 16 ++++++++++++++--
 4 files changed, 50 insertions(+), 4 deletions(-)
 create mode 100644 xenserver_post_empty_script.sh
 create mode 100644 xenserver_pre_empty_script.sh

diff --git a/cloudstackops/xenserver.py b/cloudstackops/xenserver.py
index d7f65cc..55ba84b 100644
--- a/cloudstackops/xenserver.py
+++ b/cloudstackops/xenserver.py
@@ -49,9 +49,12 @@
 # Class to handle XenServer patching
 class xenserver():
 
-    def __init__(self, ssh_user='root', threads=5):
+    def __init__(self, ssh_user='root', threads=5, pre_empty_script='xenserver_pre_empty_script.sh',
+                 post_empty_script='xenserver_post_empty_script.sh'):
         self.ssh_user = ssh_user
         self.threads = threads
+        self.pre_empty_script = pre_empty_script
+        self.post_empty_script = post_empty_script
 
     # Wait for hypervisor to become alive again
     def check_connect(self, host):
@@ -169,11 +172,16 @@ def host_evacuate(self, host):
 
     # Reboot a host when all conditions are met
     def host_reboot(self, host, halt_hypervisor=False):
-        # Disbale host
+        # Disable host
         if self.host_disable(host) is False:
             print "Error: Disabling host " + host.name + " failed."
             return False
 
+        # Execute pre-empty-script
+        if self.exec_script_on_hypervisor(host, self.pre_empty_script) is False:
+            print "Error: Executing script '" + self.pre_empty_script + "' on host " + host.name + " failed."
+            return False
+
         # Then evacuate it
         if self.host_evacuate(host) is False:
             print "Error: Evacuating host " + host.name + " failed."
@@ -185,6 +193,11 @@ def host_reboot(self, host, halt_hypervisor=False):
             return False
         print "Note: Host " + host.name + " has no VMs running, continuing"
 
+        # Execute post-empty-script
+        if self.exec_script_on_hypervisor(host, self.post_empty_script) is False:
+            print "Error: Executing script '" + self.post_empty_script + "' on host " + host.name + " failed."
+            return False
+
         # Finally reboot it
         try:
             with settings(host_string=self.ssh_user + "@" + host.ipaddress):
@@ -209,6 +222,15 @@ def host_reboot(self, host, halt_hypervisor=False):
             print "Error: Enabling host " + host.name + " failed."
             return False
 
+    # Execute script on hypervisor
+    def exec_script_on_hypervisor(self, host, script):
+        print "Note: Executing script %s on host %s.." % (script, host.name)
+        try:
+            with settings(show('output'), host_string=self.ssh_user + "@" + host.ipaddress):
+                return fab.run("bash /tmp/" + script)
+        except:
+            return False
+
     # Get VM count of a hypervisor
     def host_get_vms(self, host):
         try:
@@ -275,6 +297,12 @@ def put_scripts(self, host):
                     '/tmp/xenserver_fake_pvtools.sh', mode=0755)
                 put('xenserver_parallel_evacuate.py',
                     '/tmp/xenserver_parallel_evacuate.py', mode=0755)
+                if len(self.pre_empty_script) > 0:
+                    put(self.pre_empty_script,
+                        '/tmp/' + self.pre_empty_script, mode=0755)
+                if len(self.post_empty_script) > 0:
+                    put(self.post_empty_script,
+                        '/tmp/' + self.post_empty_script, mode=0755)
             return True
         except:
             print "Warning: Could not upload check scripts to host " + host.name + ". Continuing anyway."
diff --git a/xenserver_post_empty_script.sh b/xenserver_post_empty_script.sh
new file mode 100644
index 0000000..b7a7fbe
--- /dev/null
+++ b/xenserver_post_empty_script.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "This is the post_empty script you could customise."
\ No newline at end of file
diff --git a/xenserver_pre_empty_script.sh b/xenserver_pre_empty_script.sh
new file mode 100644
index 0000000..9211e0b
--- /dev/null
+++ b/xenserver_pre_empty_script.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "This is the pre_empty script you could customise."
\ No newline at end of file
diff --git a/xenserver_rolling_reboot.py b/xenserver_rolling_reboot.py
index c733e6a..41fd3bc 100755
--- a/xenserver_rolling_reboot.py
+++ b/xenserver_rolling_reboot.py
@@ -57,7 +57,10 @@ def handleArguments(argv):
     threads = 5
     global halt_hypervisor
     halt_hypervisor = False
-
+    global pre_empty_script
+    pre_empty_script = 'xenserver_pre_empty_script.sh'
+    global post_empty_script
+    post_empty_script = 'xenserver_post_empty_script.sh'
     # Usage message
     help = "Usage: ./" + os.path.basename(__file__) + ' [options]' + \
         '\n  --config-profile -c <profilename>\t\tSpecify the CloudMonkey profile name to ' \
@@ -68,6 +71,10 @@ def handleArguments(argv):
         '\n  --threads <nr>\t\t\t\tUse this number or concurrent migration threads ' + \
         '\n  --halt\t\t\t\t\tInstead of the default reboot, halt the hypervisor (useful in case of hardware ' \
         'upgrades) ' + \
+        '\n  --pre-empty-script\t\t\t\tBash script to run on hypervisor before starting the live migrations to empty ' \
+        'hypervisor (expected in same folder as this script)' + \
+        '\n  --post-empty-script\t\t\t\tBash script to run on hypervisor after a hypervisor has no more VMs running' \
+        '(expected in same folder as this script)' + \
         '\n  --debug\t\t\t\t\tEnable debug mode' + \
         '\n  --exec\t\t\t\t\tExecute for real' + \
         '\n  --prepare\t\t\t\t\tExecute some prepare commands'
@@ -75,7 +82,8 @@ def handleArguments(argv):
     try:
         opts, args = getopt.getopt(
             argv, "hc:n:t:p", [
-                "credentials-file=", "clustername=", "ignore-hosts=", "threads=", "halt", "debug", "exec", "prepare"])
+                "credentials-file=", "clustername=", "ignore-hosts=", "threads=", "pre-empty-script=",
+                "post-empty-script=", "halt", "debug", "exec", "prepare"])
     except getopt.GetoptError as e:
         print "Error: " + str(e)
         print help
@@ -95,6 +103,10 @@ def handleArguments(argv):
             ignoreHostList = arg
         elif opt in ("--halt"):
             halt_hypervisor = True
+        elif opt in ("--pre-empty-script"):
+            pre_empty_script = arg
+        elif opt in ("--post-empty-script"):
+            post_empty_script = arg
         elif opt in ("--debug"):
             DEBUG = 1
         elif opt in ("--exec"):

From 94725add4faf707241fe1632434a57635d98e0b3 Mon Sep 17 00:00:00 2001
From: Remi Bergsma <github@remi.nl>
Date: Thu, 4 Aug 2016 11:09:29 +0200
Subject: [PATCH 02/11] Install latest XenServer patches

---
 .gitignore                                |  3 +-
 cloudstackops/xenserver.py                | 65 ++++++++++++++++++++++-
 xenserver_patches_to_install.txt          |  0
 xenserver_post_empty_script.sh            | 55 ++++++++++++++++++-
 xenserver_pre_empty_script.sh             |  2 +-
 xenserver_rolling_reboot.py               | 25 ++++++++-
 xenserver_upload_patches_to_poolmaster.sh | 35 ++++++++++++
 7 files changed, 179 insertions(+), 6 deletions(-)
 create mode 100644 xenserver_patches_to_install.txt
 create mode 100644 xenserver_upload_patches_to_poolmaster.sh

diff --git a/.gitignore b/.gitignore
index bd4a6b3..8dad803 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,5 @@
 *.swp
 config
 *.iml
-.idea/
\ No newline at end of file
+.idea/
+xenserver_patches/
diff --git a/cloudstackops/xenserver.py b/cloudstackops/xenserver.py
index 55ba84b..fdcca45 100644
--- a/cloudstackops/xenserver.py
+++ b/cloudstackops/xenserver.py
@@ -24,6 +24,7 @@
 import sys
 import time
 import os
+import requests
 
 # Fabric
 from fabric.api import *
@@ -224,7 +225,7 @@ def host_reboot(self, host, halt_hypervisor=False):
 
     # Execute script on hypervisor
     def exec_script_on_hypervisor(self, host, script):
-        print "Note: Executing script %s on host %s.." % (script, host.name)
+        print "Note: Executing script '%s' on host %s.." % (script, host.name)
         try:
             with settings(show('output'), host_string=self.ssh_user + "@" + host.ipaddress):
                 return fab.run("bash /tmp/" + script)
@@ -336,3 +337,65 @@ def get_bond_status(self, host):
                 return fab.run("python /tmp/xenserver_check_bonds.py | awk {'print $1'} | tr -d \":\"")
         except:
             return False
+
+    # Download XenServer patch
+    def download_patch(self, url):
+        filename = url.split("/")[-1]
+
+        directory = "xenserver_patches"
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+
+        destination_file = os.getcwd() + '/' + directory + '/' + filename
+        try:
+            local_length = int(os.path.getsize(destination_file))
+        except:
+            local_length = 0
+
+        print "Note: Executing request.."
+        try:
+            response = requests.get(url, stream=True)
+            remote_length = int(response.headers.get('Content-Length', 0))
+            if not response.ok:
+                return False
+        except:
+            return False
+
+        # Do we need to download?
+        print "Note: The remote length is %s, local length is %s" % (remote_length, local_length)
+
+        if remote_length == local_length:
+            print "Note: Skipping download because file is already downloaded."
+            return True
+
+        with open(destination_file, 'wb') as handle:
+            # Download file
+            print "Note: Downloading file.."
+
+            for block in response.iter_content(1024):
+                handle.write(block)
+            return True
+
+    # Upload patches to poolmaster
+    def put_patches_to_poolmaster(self, host):
+        print "Note: Uploading patches to poolmaster.."
+        try:
+            with settings(host_string=self.ssh_user + "@" + host.ipaddress):
+                run('rm -rf /root/xenserver_patches/')
+                run('mkdir -p /root/xenserver_patches')
+                put('xenserver_patches/*', '/root/xenserver_patches')
+                put('xenserver_upload_patches_to_poolmaster.sh',
+                    '/root/xenserver_patches/xenserver_upload_patches_to_poolmaster.sh', mode=0755)
+            return True
+        except:
+            print "Warning: Could not upload patches to host " + host.name + "."
+            return False
+
+    # Upload patches to XenServer
+    def upload_patches_to_xenserver(self, host):
+        print "Note: We're uploading the patches to XenServer"
+        try:
+            with settings(show('output'), host_string=self.ssh_user + "@" + host.ipaddress):
+                return fab.run("bash /root/xenserver_patches/xenserver_upload_patches_to_poolmaster.sh")
+        except:
+            return False
diff --git a/xenserver_patches_to_install.txt b/xenserver_patches_to_install.txt
new file mode 100644
index 0000000..e69de29
diff --git a/xenserver_post_empty_script.sh b/xenserver_post_empty_script.sh
index b7a7fbe..b6319bf 100644
--- a/xenserver_post_empty_script.sh
+++ b/xenserver_post_empty_script.sh
@@ -1,3 +1,56 @@
 #!/usr/bin/env bash
 
-echo "This is the post_empty script you could customise."
\ No newline at end of file
+echo "This is the post_empty script you could customise."
+
+#echo "Downgrading openvswitch RPM to the XenServer default"
+#rpm -Uvh http://10.200.10.10/software/xenserver/openvswitch-1.4.6-143.9926.i386.rpm --force --nodeps
+
+echo "Applying patches"
+HOST_UUID=$(xe host-list name-label=${HOSTNAME} --minimal)
+
+# Check for 6.5
+cat /etc/redhat-release | grep "6.5"
+if [ $? -eq 0 ]; then
+    SERVICE_PACK_PATCH=XS65ESP1
+fi
+# Check for 6.2
+cat /etc/redhat-release | grep "6.2"
+if [ $? -eq 0 ]; then
+    SERVICE_PACK_PATCH=XS62ESP1
+fi
+
+# First apply SP1
+xe patch-list name-label=${SERVICE_PACK_PATCH} params=hosts --minimal | tr ',' '\n' | grep ${HOST_UUID}
+if [ $? -eq 0 ]; then
+   echo Service Pack ${SERVICE_PACK_PATCH} is already installed, skipping.
+else
+   echo Installing ${SERVICE_PACK_PATCH}...
+   PATCH_UUID=$(xe patch-list name-label=${SERVICE_PACK_PATCH} | grep uuid | sed -e 's/^.*: //g')
+   if [ ${PATCH_UUID} ]; then
+       xe patch-apply uuid=${PATCH_UUID} host-uuid=${HOST_UUID}
+   fi
+fi
+
+# Apply any other available patch
+XEN_ALL_PATCHES=$(xe patch-list params=name-label --minimal | tr ',' '\n' )
+XEN_INSTALLED_PATCHES=$(xe patch-list hosts:contains=${HOST_UUID} params=name-label --minimal | tr ',' '\n' )
+
+for patch in ${XEN_ALL_PATCHES}; do
+    echo "Checking patch " ${patch}
+
+    # Check if already included
+    echo ${XEN_INSTALLED_PATCHES} | grep ${patch} 2>&1 >/dev/null
+    if [ $? -eq 0 ]; then
+       echo Patch ${patch} is already installed, skipping.
+    else
+       echo Installing $patch...
+       PATCH_UUID=$(xe patch-list name-label=${patch}| grep uuid | sed -e 's/^.*: //g')
+       if [ ${PATCH_UUID} ]; then
+           xe patch-apply uuid=${PATCH_UUID} host-uuid=${HOST_UUID}
+       fi
+    fi
+done
+
+echo "Upgrading drivers"
+yum -y install bnx2x-* fnic* qla2* glnic* qlge* tg3* hpsa* openvswitch-modules-xen*
+yum -y upgrade nicira-ovs-hypervisor-node
diff --git a/xenserver_pre_empty_script.sh b/xenserver_pre_empty_script.sh
index 9211e0b..fca53ec 100644
--- a/xenserver_pre_empty_script.sh
+++ b/xenserver_pre_empty_script.sh
@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-echo "This is the pre_empty script you could customise."
\ No newline at end of file
+echo "This is the pre_empty script you could customise."
diff --git a/xenserver_rolling_reboot.py b/xenserver_rolling_reboot.py
index 41fd3bc..c7d986e 100755
--- a/xenserver_rolling_reboot.py
+++ b/xenserver_rolling_reboot.py
@@ -61,6 +61,8 @@ def handleArguments(argv):
     pre_empty_script = 'xenserver_pre_empty_script.sh'
     global post_empty_script
     post_empty_script = 'xenserver_post_empty_script.sh'
+    global patch_list_file
+    patch_list_file = 'xenserver_patches_to_install.txt'
     # Usage message
     help = "Usage: ./" + os.path.basename(__file__) + ' [options]' + \
         '\n  --config-profile -c <profilename>\t\tSpecify the CloudMonkey profile name to ' \
@@ -74,6 +76,7 @@ def handleArguments(argv):
         '\n  --pre-empty-script\t\t\t\tBash script to run on hypervisor before starting the live migrations to empty ' \
         'hypervisor (expected in same folder as this script)' + \
         '\n  --post-empty-script\t\t\t\tBash script to run on hypervisor after a hypervisor has no more VMs running' \
+        '\n  --patch-list-file\t\t\t\tText file with URLs of patches to download and install. One per line. ' \
         '(expected in same folder as this script)' + \
         '\n  --debug\t\t\t\t\tEnable debug mode' + \
         '\n  --exec\t\t\t\t\tExecute for real' + \
@@ -83,7 +86,7 @@ def handleArguments(argv):
         opts, args = getopt.getopt(
             argv, "hc:n:t:p", [
                 "credentials-file=", "clustername=", "ignore-hosts=", "threads=", "pre-empty-script=",
-                "post-empty-script=", "halt", "debug", "exec", "prepare"])
+                "post-empty-script=", "patch-list-file=", "halt", "debug", "exec", "prepare"])
     except getopt.GetoptError as e:
         print "Error: " + str(e)
         print help
@@ -107,6 +110,8 @@ def handleArguments(argv):
             pre_empty_script = arg
         elif opt in ("--post-empty-script"):
             post_empty_script = arg
+        elif opt in ("--patch-list-file"):
+            patch_list_file = arg
         elif opt in ("--debug"):
             DEBUG = 1
         elif opt in ("--exec"):
@@ -137,7 +142,7 @@ def handleArguments(argv):
 c = cloudstackops.CloudStackOps(DEBUG, DRYRUN)
 
 # Init XenServer class
-x = xenserver.xenserver('root', threads)
+x = xenserver.xenserver('root', threads, pre_empty_script, post_empty_script)
 c.xenserver = x
 
 # make credentials file known to our class
@@ -212,7 +217,10 @@ def handleArguments(argv):
     print "  - Turn OFF XenServer poolHA for " + clustername
     print "  - For any hypervisor it will do this (poolmaster " + poolmaster.name + " first):"
     print "      - put it to Disabled aka Maintenance in XenServer"
+    print "      - download the patches in file --patch-list-file '" + patch_list_file + "'"
+    print "      - execute the --pre-empty-script script '" + pre_empty_script + "' on the hypervisor"
     print "      - live migrate all VMs off of it using XenServer evacuate command"
+    print "      - execute the --post-empty-script script '" + post_empty_script + "' on the hypervisor"
     print "      - when empty, it will reboot the hypervisor (halting is " + str(halt_hypervisor) + ")"
     print "      - will wait for it to come back online (checks SSH connection)"
     print "      - set the hypervisor to Enabled in XenServer"
@@ -261,6 +269,19 @@ def handleArguments(argv):
         disconnect_all()
         sys.exit(1)
 
+    # Download all XenServer patches
+    print "Note: Reading patches list '%s'" % patch_list_file
+    with open(patch_list_file) as file_pointer:
+        patches = file_pointer.read().splitlines()
+
+    for patch_url in patches:
+        print "Note: Processing patch '%s'" % patch_url
+        x.download_patch(patch_url)
+
+    # Upload the patches to poolmaster, then to XenServer
+    x.put_patches_to_poolmaster(poolmaster)
+    x.upload_patches_to_xenserver(poolmaster)
+
     # Migrate all VMs off of pool master
     vm_count = x.host_get_vms(poolmaster)
     if vm_count:
diff --git a/xenserver_upload_patches_to_poolmaster.sh b/xenserver_upload_patches_to_poolmaster.sh
new file mode 100644
index 0000000..a4a53e6
--- /dev/null
+++ b/xenserver_upload_patches_to_poolmaster.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+if [ "$(xe host-list name-label=$HOSTNAME --minimal)" == "$(xe pool-list params=master --minimal)" ]
+then
+    echo "We are the poolmaster, so let's process the patches."
+
+    echo "Unzipping patches"
+    cd /root/xenserver_patches
+    for file in $(ls /root/xenserver_patches/*.zip); do unzip ${file}; done
+
+    echo "Deleting what we don't need"
+    rm -f /root/xenserver_patches/*.zip /root/xenserver_patches/*.bz2
+
+    # Check if already included
+    XEN_ALL_PATCHES=$(xe patch-list params=name-label --minimal | tr ',' '\n' )
+    for patch in $(ls *.xsupdate)
+    do
+        echo "Processing patch file ${patch}"
+
+        patchname=${patch%.*}
+        echo "Checking if ${patchname} from file ${patch} is already uploaded"
+        echo ${XEN_ALL_PATCHES} | tr ' ' '\n' | grep ^${patchname}$ 2>&1 >/dev/null
+        if [ $? -eq 0 ]; then
+           echo Patch ${patch} is already installed, skipping.
+        else
+            echo "Uploading patch ${patchname}"
+            xe patch-upload file-name=${patch}
+            if [ $? -gt 0 ]; then
+                echo "Uploading failed, continuing with other patches"
+            fi
+        fi
+    done
+else
+    echo "We are NOT poolmaster, so skipping uploading patches"
+fi

From 9d50d9da80db60fabed7ba981d1cad556d446f1f Mon Sep 17 00:00:00 2001
From: Remi Bergsma <github@remi.nl>
Date: Mon, 8 Aug 2016 10:33:26 +0200
Subject: [PATCH 03/11] Fix openswitch after patching back to newest available

---
 xenserver_post_empty_script.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/xenserver_post_empty_script.sh b/xenserver_post_empty_script.sh
index b6319bf..398a2ab 100644
--- a/xenserver_post_empty_script.sh
+++ b/xenserver_post_empty_script.sh
@@ -54,3 +54,6 @@ done
 echo "Upgrading drivers"
 yum -y install bnx2x-* fnic* qla2* glnic* qlge* tg3* hpsa* openvswitch-modules-xen*
 yum -y upgrade nicira-ovs-hypervisor-node
+
+echo "Fixing openvswitch installation back to the newest"
+yum reinstall openvswitch
\ No newline at end of file

From e2aacbf8a5ef8529e8b94b0e75f0d499a72a964f Mon Sep 17 00:00:00 2001
From: Remi Bergsma <github@remi.nl>
Date: Mon, 8 Aug 2016 11:03:30 +0200
Subject: [PATCH 04/11] Make exception for HPSA driver on G7

---
 xenserver_post_empty_script.sh | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/xenserver_post_empty_script.sh b/xenserver_post_empty_script.sh
index 398a2ab..a62c313 100644
--- a/xenserver_post_empty_script.sh
+++ b/xenserver_post_empty_script.sh
@@ -3,7 +3,7 @@
 echo "This is the post_empty script you could customise."
 
 #echo "Downgrading openvswitch RPM to the XenServer default"
-#rpm -Uvh http://10.200.10.10/software/xenserver/openvswitch-1.4.6-143.9926.i386.rpm --force --nodeps
+rpm -Uvh http://10.200.10.10/software/xenserver/openvswitch-1.4.6-143.9926.i386.rpm --force --nodeps
 
 echo "Applying patches"
 HOST_UUID=$(xe host-list name-label=${HOSTNAME} --minimal)
@@ -52,8 +52,12 @@ for patch in ${XEN_ALL_PATCHES}; do
 done
 
 echo "Upgrading drivers"
-yum -y install bnx2x-* fnic* qla2* glnic* qlge* tg3* hpsa* openvswitch-modules-xen*
-yum -y upgrade nicira-ovs-hypervisor-node
+yum -y install bnx2x-* fnic* qla2* glnic* qlge* tg3* openvswitch-modules-xen*
 
-echo "Fixing openvswitch installation back to the newest"
-yum reinstall openvswitch
\ No newline at end of file
+SYSTEM_HARDWARE=$(dmidecode -s system-product-name | grep -v "#")
+if [[ "${SYSTEM_HARDWARE}" == "ProLiant DL380 G7" ]]; then
+    echo "Skip HPSA on HP ProLiant DL380 G7 or else the box won't boot"
+else
+    yum -y install hpsa*
+fi
+yum -y upgrade nicira-ovs-hypervisor-node

From 60a43099784e596b54c40f3c1d46c571a47f585c Mon Sep 17 00:00:00 2001
From: Remi Bergsma <github@remi.nl>
Date: Mon, 8 Aug 2016 14:48:14 +0200
Subject: [PATCH 05/11] Make check_xapi warn-only

---
 cloudstackops/xenserver.py     |  5 ++---
 xenserver_post_empty_script.sh | 12 ++++++++++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/cloudstackops/xenserver.py b/cloudstackops/xenserver.py
index fdcca45..a2969b6 100644
--- a/cloudstackops/xenserver.py
+++ b/cloudstackops/xenserver.py
@@ -85,9 +85,8 @@ def check_connect(self, host):
     # Check if we can use xapi
     def check_xapi(self, host):
         try:
-            with settings(host_string=self.ssh_user + "@" + host.ipaddress):
-                with warn_only():
-                    result = fab.run("xe host-enable host=" + host.name)
+            with settings(warn_only=True, host_string=self.ssh_user + "@" + host.ipaddress):
+                result = fab.run("xe host-enable host=" + host.name)
                 if result.return_code == 0:
                     return True
                 else:
diff --git a/xenserver_post_empty_script.sh b/xenserver_post_empty_script.sh
index a62c313..0732e1f 100644
--- a/xenserver_post_empty_script.sh
+++ b/xenserver_post_empty_script.sh
@@ -53,11 +53,23 @@ done
 
 echo "Upgrading drivers"
 yum -y install bnx2x-* fnic* qla2* glnic* qlge* tg3* openvswitch-modules-xen*
+if [ $? -eq 0 ]; then
+    echo "Yum commnand returned non-zero"
+    exit 1
+fi
 
 SYSTEM_HARDWARE=$(dmidecode -s system-product-name | grep -v "#")
 if [[ "${SYSTEM_HARDWARE}" == "ProLiant DL380 G7" ]]; then
     echo "Skip HPSA on HP ProLiant DL380 G7 or else the box won't boot"
 else
     yum -y install hpsa*
+    if [ $? -eq 0 ]; then
+    echo "Yum commnand returned non-zero"
+    exit 1
+fi
 fi
 yum -y upgrade nicira-ovs-hypervisor-node
+if [ $? -eq 0 ]; then
+    echo "Yum commnand returned non-zero"
+    exit 1
+fi
\ No newline at end of file

From 67d7ee92bb0dc93b3f8ec9827e0be5926304e1dd Mon Sep 17 00:00:00 2001
From: Remi Bergsma <github@remi.nl>
Date: Tue, 9 Aug 2016 15:58:54 +0200
Subject: [PATCH 06/11] Allow to specify scripts from any folder

---
 cloudstackops/xenserver.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cloudstackops/xenserver.py b/cloudstackops/xenserver.py
index a2969b6..013961a 100644
--- a/cloudstackops/xenserver.py
+++ b/cloudstackops/xenserver.py
@@ -224,6 +224,7 @@ def host_reboot(self, host, halt_hypervisor=False):
 
     # Execute script on hypervisor
     def exec_script_on_hypervisor(self, host, script):
+        host = host.split('/')[-1]
         print "Note: Executing script '%s' on host %s.." % (script, host.name)
         try:
             with settings(show('output'), host_string=self.ssh_user + "@" + host.ipaddress):
@@ -299,10 +300,10 @@ def put_scripts(self, host):
                     '/tmp/xenserver_parallel_evacuate.py', mode=0755)
                 if len(self.pre_empty_script) > 0:
                     put(self.pre_empty_script,
-                        '/tmp/' + self.pre_empty_script, mode=0755)
+                        '/tmp/' + self.pre_empty_script.split('/')[-1], mode=0755)
                 if len(self.post_empty_script) > 0:
                     put(self.post_empty_script,
-                        '/tmp/' + self.post_empty_script, mode=0755)
+                        '/tmp/' + self.post_empty_script.split('/')[-1], mode=0755)
             return True
         except:
             print "Warning: Could not upload check scripts to host " + host.name + ". Continuing anyway."

From 2e1c8790edae9c303f0358ea53371e332e9c9956 Mon Sep 17 00:00:00 2001
From: Remi Bergsma <github@remi.nl>
Date: Tue, 9 Aug 2016 16:04:35 +0200
Subject: [PATCH 07/11] Do not require space in ignore-hosts parameter

---
 xenserver_rolling_reboot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xenserver_rolling_reboot.py b/xenserver_rolling_reboot.py
index c7d986e..184061d 100755
--- a/xenserver_rolling_reboot.py
+++ b/xenserver_rolling_reboot.py
@@ -125,7 +125,7 @@ def handleArguments(argv):
 
     # Ignore host list
     if len(ignoreHostList) > 0:
-        ignoreHosts = ignoreHostList.split(", ")
+        ignoreHosts = ignoreHostList.replace(' ', '').split(",")
     else:
         ignoreHosts = []
 

From 296cc6d9cd498f5f8f6ded5a8a6f77063ccaa873 Mon Sep 17 00:00:00 2001
From: Remi Bergsma <github@remi.nl>
Date: Tue, 9 Aug 2016 16:07:10 +0200
Subject: [PATCH 08/11] Wait a bit longer until we come up with a better
 solution

---
 xenserver_rolling_reboot.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/xenserver_rolling_reboot.py b/xenserver_rolling_reboot.py
index 184061d..1485fbb 100755
--- a/xenserver_rolling_reboot.py
+++ b/xenserver_rolling_reboot.py
@@ -307,8 +307,8 @@ def handleArguments(argv):
         disconnect_all()
         sys.exit(1)
 
-    print "Note: Waiting 30s to allow all hosts connect.."
-    time.sleep(30)
+    print "Note: Waiting 60s to allow all hosts connect.."
+    time.sleep(60)
 
 else:
         print "Warning: Skipping " + poolmaster.name + " due to --ignore-hosts setting"

From 3ce57a1febae8490ebc9113bf1ef257e51f2e0c4 Mon Sep 17 00:00:00 2001
From: Remi Bergsma <github@remi.nl>
Date: Wed, 10 Aug 2016 11:11:21 +0200
Subject: [PATCH 09/11] By default do not preserve downloads

---
 xenserver_rolling_reboot.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/xenserver_rolling_reboot.py b/xenserver_rolling_reboot.py
index 1485fbb..08cfcd6 100755
--- a/xenserver_rolling_reboot.py
+++ b/xenserver_rolling_reboot.py
@@ -28,6 +28,7 @@
 import time
 import os
 import getopt
+import glob
 from cloudstackops import cloudstackops
 from cloudstackops import xenserver
 # Fabric
@@ -63,6 +64,9 @@ def handleArguments(argv):
     post_empty_script = 'xenserver_post_empty_script.sh'
     global patch_list_file
     patch_list_file = 'xenserver_patches_to_install.txt'
+    global preserve_downloads
+    preserve_downloads = False
+
     # Usage message
     help = "Usage: ./" + os.path.basename(__file__) + ' [options]' + \
         '\n  --config-profile -c <profilename>\t\tSpecify the CloudMonkey profile name to ' \
@@ -78,6 +82,7 @@ def handleArguments(argv):
         '\n  --post-empty-script\t\t\t\tBash script to run on hypervisor after a hypervisor has no more VMs running' \
         '\n  --patch-list-file\t\t\t\tText file with URLs of patches to download and install. One per line. ' \
         '(expected in same folder as this script)' + \
+        '\n  --preserve-downloads\t\t\t\tPreserve downloads instead of wiping them and downloading again.' + \
         '\n  --debug\t\t\t\t\tEnable debug mode' + \
         '\n  --exec\t\t\t\t\tExecute for real' + \
         '\n  --prepare\t\t\t\t\tExecute some prepare commands'
@@ -86,7 +91,7 @@ def handleArguments(argv):
         opts, args = getopt.getopt(
             argv, "hc:n:t:p", [
                 "credentials-file=", "clustername=", "ignore-hosts=", "threads=", "pre-empty-script=",
-                "post-empty-script=", "patch-list-file=", "halt", "debug", "exec", "prepare"])
+                "post-empty-script=", "patch-list-file=", "preserve-downloads", "halt", "debug", "exec", "prepare"])
     except getopt.GetoptError as e:
         print "Error: " + str(e)
         print help
@@ -112,6 +117,8 @@ def handleArguments(argv):
             post_empty_script = arg
         elif opt in ("--patch-list-file"):
             patch_list_file = arg
+        elif opt in ("--preserve-downloads"):
+            preserve_downloads = True
         elif opt in ("--debug"):
             DEBUG = 1
         elif opt in ("--exec"):
@@ -218,6 +225,7 @@ def handleArguments(argv):
     print "  - For any hypervisor it will do this (poolmaster " + poolmaster.name + " first):"
     print "      - put it to Disabled aka Maintenance in XenServer"
     print "      - download the patches in file --patch-list-file '" + patch_list_file + "'"
+    print "         (preserve downloads is set to " + str(preserve_downloads) + ")"
     print "      - execute the --pre-empty-script script '" + pre_empty_script + "' on the hypervisor"
     print "      - live migrate all VMs off of it using XenServer evacuate command"
     print "      - execute the --post-empty-script script '" + post_empty_script + "' on the hypervisor"
@@ -270,6 +278,13 @@ def handleArguments(argv):
         sys.exit(1)
 
     # Download all XenServer patches
+    if not preserve_downloads:
+        print "Note: Deleting previously downloaded patches"
+        files = glob.glob('xenserver_patches/*.zip')
+        for f in files:
+            print "Note: Removing previously downloaded patch " + f
+            os.remove(f)
+
     print "Note: Reading patches list '%s'" % patch_list_file
     with open(patch_list_file) as file_pointer:
         patches = file_pointer.read().splitlines()

From ba275788019f9b403ccc0d10e507777aee72409b Mon Sep 17 00:00:00 2001
From: Remi Bergsma <github@remi.nl>
Date: Wed, 10 Aug 2016 11:33:47 +0200
Subject: [PATCH 10/11] Fix bug in script name parsing

---
 cloudstackops/xenserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudstackops/xenserver.py b/cloudstackops/xenserver.py
index 013961a..19be206 100644
--- a/cloudstackops/xenserver.py
+++ b/cloudstackops/xenserver.py
@@ -224,7 +224,7 @@ def host_reboot(self, host, halt_hypervisor=False):
 
     # Execute script on hypervisor
     def exec_script_on_hypervisor(self, host, script):
-        host = host.split('/')[-1]
+        script = script.split('/')[-1]
         print "Note: Executing script '%s' on host %s.." % (script, host.name)
         try:
             with settings(show('output'), host_string=self.ssh_user + "@" + host.ipaddress):

From 2332eb8a676cbfb0ada4a37bb407c3689fad8c56 Mon Sep 17 00:00:00 2001
From: Remi Bergsma <github@remi.nl>
Date: Wed, 10 Aug 2016 12:13:00 +0200
Subject: [PATCH 11/11] Bump timer

---
 cloudstackops/xenserver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cloudstackops/xenserver.py b/cloudstackops/xenserver.py
index 19be206..c1f05a1 100644
--- a/cloudstackops/xenserver.py
+++ b/cloudstackops/xenserver.py
@@ -78,8 +78,8 @@ def check_connect(self, host):
         # Remove progress indication
         sys.stdout.write("\033[F")
         print "Note: Host " + host.name + " is able to do XE stuff again!                                  "
-        print "Note: Waiting 30s to allow the hypervisor to connect.."
-        time.sleep(30)
+        print "Note: Waiting 60s to allow the hypervisor to connect.."
+        time.sleep(60)
         return True
 
     # Check if we can use xapi