Rebase to resolve tempest run error

README.rst

@@ -252,4 +252,4 @@ You can file bugs on the storyboard devstack-gate project::
 
     https://storyboard.openstack.org/#!/project/712
 
-And you can chat with us on Freenode in #openstack-dev or #openstack-infra.
+And you can chat with us on Freenode in #openstack-qa or #openstack-infra.

devstack-vm-gate-wrap.sh

@@ -37,6 +37,11 @@ source $WORKSPACE/devstack-gate/functions.sh
 
 start_timer
 
+# Save the PROJECTS variable as it was passed in.  This is needed for reproduce.sh
+# incase the job definition contains items that are not in the "global" list
+# below.
+# See: https://bugs.launchpad.net/openstack-gate/+bug/1544827
+JOB_PROJECTS="$PROJECTS"
 PROJECTS="openstack-infra/devstack-gate $PROJECTS"
 PROJECTS="openstack-dev/devstack $PROJECTS"
 PROJECTS="openstack-dev/pbr $PROJECTS"
@@ -71,10 +76,12 @@ PROJECTS="openstack/neutron-vpnaas $PROJECTS"
 PROJECTS="openstack/nova $PROJECTS"
 PROJECTS="openstack/os-apply-config $PROJECTS"
 PROJECTS="openstack/os-brick $PROJECTS"
+PROJECTS="openstack/os-client-config $PROJECTS"
 PROJECTS="openstack/os-cloud-config $PROJECTS"
 PROJECTS="openstack/os-collect-config $PROJECTS"
 PROJECTS="openstack/os-net-config $PROJECTS"
 PROJECTS="openstack/os-refresh-config $PROJECTS"
+PROJECTS="openstack/osc-lib $PROJECTS"
 PROJECTS="openstack/oslo.cache $PROJECTS"
 PROJECTS="openstack/oslo.concurrency $PROJECTS"
 PROJECTS="openstack/oslo.config $PROJECTS"
@@ -163,8 +170,15 @@ export DEVSTACK_GATE_TEMPEST_DISABLE_TENANT_ISOLATION=${DEVSTACK_GATE_TEMPEST_DI
 # Set to 1 to enable Cinder secure delete.
 # False by default to avoid dd problems on Precise.
 # https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1023755
+# TODO(mriedem): CINDER_SECURE_DELETE is deprecated in devstack as of liberty.
+# Remove after kilo-eol happens in devstack.
 export DEVSTACK_CINDER_SECURE_DELETE=${DEVSTACK_CINDER_SECURE_DELETE:-0}
 
+# Should cinder perform secure deletion of volumes?
+# Defaults to none to avoid bug 1023755. Can also be set to zero or shred.
+# Only applicable to stable/liberty+ devstack.
+export DEVSTACK_CINDER_VOLUME_CLEAR=${DEVSTACK_CINDER_VOLUME_CLEAR:-none}
+
 # Set to 1 to run neutron instead of nova network
 # Only applicable to master branch
 export DEVSTACK_GATE_NEUTRON=${DEVSTACK_GATE_NEUTRON:-0}
@@ -269,6 +283,9 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then
                 export GRENADE_NEW_BRANCH="stable/liberty"
             elif [[ "$GRENADE_BASE_BRANCH" == "stable/liberty" ]]; then
                 export GRENADE_OLD_BRANCH="stable/liberty"
+                export GRENADE_NEW_BRANCH="stable/mitaka"
+            elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then
+                export GRENADE_OLD_BRANCH="stable/mitaka"
                 export GRENADE_NEW_BRANCH="$GIT_BRANCH"
             fi
             ;;
@@ -296,8 +313,11 @@ if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then
             elif [[ "$GRENADE_BASE_BRANCH" == "stable/liberty" ]]; then
                 export GRENADE_OLD_BRANCH="stable/kilo"
                 export GRENADE_NEW_BRANCH="stable/liberty"
-            else # master
+            elif [[ "$GRENADE_BASE_BRANCH" == "stable/mitaka" ]]; then
                 export GRENADE_OLD_BRANCH="stable/liberty"
+                export GRENADE_NEW_BRANCH="stable/mitaka"
+            else # master
+                export GRENADE_OLD_BRANCH="stable/mitaka"
                 export GRENADE_NEW_BRANCH="$GIT_BRANCH"
             fi
             ;;
@@ -416,7 +436,7 @@ fi
 if ! function_exists "gate_hook"; then
     # the command we use to run the gate
     function gate_hook {
-        $BASE/new/devstack-gate/devstack-vm-gate.sh
+        $WORKSPACE/devstack-gate/devstack-vm-gate.sh
     }
     export -f gate_hook
 fi
@@ -434,7 +454,9 @@ set -x
 # Install ansible
 sudo -H pip install virtualenv
 virtualenv /tmp/ansible
-/tmp/ansible/bin/pip install ansible==$ANSIBLE_VERSION
+# NOTE(emilien): workaround to avoid installing cryptography
+# https://github.com/ansible/ansible/issues/15665
+sudo -H /tmp/ansible/bin/pip install paramiko==1.16.0 ansible==$ANSIBLE_VERSION
 export ANSIBLE=/tmp/ansible/bin/ansible
 
 # Write inventory file with groupings
@@ -468,7 +490,7 @@ $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m file \
     -a "path='$WORKSPACE/logs' state=directory"
 
 # Record a file to reproduce this build
-reproduce
+reproduce "$JOB_PROJECTS"
 
 # Run ansible to do setup_host on all nodes.
 echo "Setting up the hosts"
@@ -552,15 +574,23 @@ fi
 # devstack-vm-gate-wrap.sh will not automagically run the hooks on each node.
 # Run pre test hook if we have one
 with_timeout call_hook_if_defined "pre_test_hook"
+GATE_RETVAL=$?
+if [ $GATE_RETVAL -ne 0 ]; then
+    echo "ERROR: the pre-test setup script run by this job failed - exit code: $GATE_RETVAL"
+fi
 
 # Run the gate function
-echo "Running gate_hook"
-with_timeout "gate_hook"
-GATE_RETVAL=$?
+if [ $GATE_RETVAL -eq 0 ]; then
+    echo "Running gate_hook"
+    with_timeout "gate_hook"
+    GATE_RETVAL=$?
+    if [ $GATE_RETVAL -ne 0 ]; then
+        echo "ERROR: the main setup script run by this job failed - exit code: $GATE_RETVAL"
+    fi
+fi
 RETVAL=$GATE_RETVAL
 
 if [ $GATE_RETVAL -ne 0 ]; then
-    echo "ERROR: the main setup script run by this job failed - exit code: $GATE_RETVAL"
     echo "    please look at the relevant log files to determine the root cause"
     echo "Running devstack worlddump.py"
     sudo $BASE/new/devstack/tools/worlddump.py -d $BASE/logs