jenkins-dell-crowbar-admin-node

#!/bin/bash
#
# $1 = test repo number
#
env
# JENKINS COMMON
[ -e $(dirname $0)/jenkins-deb-common ] || exit 1
. $(dirname $0)/jenkins-deb-common
jenkins_init
jenkins_set_vars

# IPMI VARIABLES
[ -e ~jenkins/.ipmirc ] || exit 1
. ~jenkins/.ipmirc

# Setup global variables
SSH_OPTS='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
PROPOSAL_NAME="jenkins"

function ipmi_restart() {
    set +x
    # $1 - IP
    ip=$1
    log "Running ipmi_restart using:"
    log " ip: ${ip}"

    # First check the status
    # This will return "Chassis Power is off|on"
    status=$(ssh openstack@${IPMIHOST} "ipmitool -I lanplus -H ${ip} -U ${DRACUSER} -P ${DRACPASS} chassis power status | cut -d ' ' -f4")
    if [ $status == "on" ]; then
        # Power is on, we can powercycle
        # Gonna wilk it up here
        for i in $(seq 1 4); do
            ssh openstack@${IPMIHOST} "ipmitool -I lanplus -H ${ip} -U ${DRACUSER} -P ${DRACPASS} chassis power cycle"
        done
    else
        # Power is off, we can poweron
        # Gonna wilk it up here
        for i in $(seq 1 4); do
            ssh openstack@${IPMIHOST} "ipmitool -I lanplus -H ${ip} -U ${DRACUSER} -P ${DRACPASS} chassis power on"
        done
    fi

    # we need to sleep for 5 seconds and make sure the box came back online
    sleep 15s
    status=$(ssh openstack@${IPMIHOST} "ipmitool -I lanplus -H ${ip} -U ${DRACUSER} -P ${DRACPASS} chassis power status | cut -d ' ' -f4")
    if [ $status == "off" ]; then
        # Server did not come back online
        log "Unable to powercycle server successfully"
        exit 1
    fi
    set -x
}

function ipmi_set_pxe() {
    set +x
    # $1 - IP
    ip=$1
    log "Running ipmi_set_pxe using:"
    log " ip: ${ip}"
    # Gonna wilk it up here
    for i in $(seq 1 4); do
        ssh openstack@${IPMIHOST} "ipmitool -I lanplus -H ${ip} -U ${DRACUSER} -P ${DRACPASS} chassis bootdev pxe"
    done
    set -x
}

function ipmi_pxe_reboot() {
    # $1 - IP
    ip=$1
    log "Running ipmi_pxe_reboot using:"
    log " ip: ${ip}"
    ipmi_set_pxe ${ip}
    ipmi_restart ${ip}
}

function ipmi_poweroff() {
    set +x
    # $1 - IP
    ip=$1
    log "Running ipmi_poweroff using:"
    log " ip: ${ip}"
    # Gonna wilk it up here
    for i in $(seq 1 4); do
        ssh openstack@${IPMIHOST} "ipmitool -I lanplus -H ${ip} -U ${DRACUSER} -P ${DRACPASS} chassis power off"
    done
    set -x
}

function ipmi_poweron() {
    set +x
    # $1 - IP
    ip=$1
    log "Running ipmi_poweron using:"
    log " ip: ${ip}"
    # Gonna wilk it up here
    for i in $(seq 1 4); do
        ssh openstack@${IPMIHOST} "ipmitool -I lanplus -H ${ip} -U ${DRACUSER} -P ${DRACPASS} chassis power on"
    done
    set -x
}

function crowbar_proposal() {
    # $1 - Service Name
    # $2 - Action (create|commit)
    service=$1
    action=$2
    cmd="/opt/dell/bin/crowbar_${service} -U ${CROWBAR_USER} -P ${CROWBAR_PASS}"
    log "Executing crowbar_proposal using:"
    log " Service: ${service}"
    log " Action: ${action}"

    if ! ( ssh ${SSH_OPTS} crowbar@${HEADNODE} "${cmd} proposal ${action} ${PROPOSAL_NAME}" ); then
        log "Unable to ${action} the ${service} Proposal"
        exit 1
    fi
}

function crowbar_proposal_status() {
    # $1 - Service Name
    # $2 - Wait Time
    service=$1
    wait_timer=${2:-15} # Default to 15 minutes if no wait_time provided
    cmd="/opt/dell/bin/crowbar_${service} -U ${CROWBAR_USER} -P ${CROWBAR_PASS}"
    log "Executing crowbar_proposal using:"
    log " Service: ${service}"
    log " Wait Time: ${wait_timer}"

    count=1
    while [ $count -lt $wait_timer ]; do
        count=$(( count + 1 ))
        sleep 60s
        if ( ssh ${SSH_OPTS} crowbar@${HEADNODE} "${cmd} proposal show ${PROPOSAL_NAME} | grep crowbar-status | grep success" ); then
            log "${service} proposal sucessfully applied"
            break
        fi
        if [ $count == $wait_timer ]; then
            log "${service} proposal not applied"
            exit 1
        fi
    done
}

# SETUP ISO NAME
if [ -z "$NOVA_RELEASE" ] || [ -z "$MILESTONE" ] || [ -z "$DISTRELEASE" ]; then
    ISONAME="crowbar.iso"
else
    BINARY_BUILD_RELEASE=`echo $DISTRELEASE | cut -d'-' -f2`
    ISONAME="crowbar-rcb-${NOVA_RELEASE}-${MILESTONE}-${BINARY_BUILD_RELEASE}.iso"
fi

# EXEC CROWBAR BUILD
set +x
ssh openstack@${ISODEST} "sudo /home/openstack/incoming/fixup_crowbar.sh ${ISONAME}"
set -x

# PXE REBOOT HEADNODE
ipmi_pxe_reboot ${HEADNODE_DRAC_IP}

# POWER OFF CONTROLLER NODE / COMPUTE NODE
ipmi_poweroff ${CONTROLLER_DRAC_IP}
ipmi_poweroff ${COMPUTE_DRAC_IP}

# WAIT TO ENSURE TARGET GOES DOWN FOR REBOOT
sleep 60s

# WAIT FOR ADMIN NODE TO HAVE SSH
# This takes ~20 mins
count=1
while [ $count -lt 25 ]; do
    count=$(( count + 1 ))
    sleep 60s
    if ( nc ${HEADNODE} 22 -w 1 -q 0 < /dev/null ); then
        break
    fi
    if [ $count -eq 25 ]; then
        log "Server is not network accessible"
        exit 1
    fi
done

# Give crowbar 10 minutes to install once SSH is available
count=1
while [ $count -lt 10 ]; do
    count=$(( count + 1 ))
    sleep 60s
    if ( ssh ${SSH_OPTS} crowbar@${HEADNODE} 'sudo -i /opt/dell/bin/crowbar_node_state status | grep -e crowbar | grep -e Ready' ); then
        break
    fi
    if [ $count == 10 ]; then
        log "Installation of Crowbar Headnode failed"
        exit 1
    fi
done

sleep 120s

# POWER ON CONTROLLER NODE 
ipmi_set_pxe ${CONTROLLER_DRAC_IP}
ipmi_poweron ${CONTROLLER_DRAC_IP}

# Bring up the dedicated Compute Node
ipmi_set_pxe ${COMPUTE_DRAC_IP}
ipmi_poweron ${COMPUTE_DRAC_IP}

# WAIT TO ENSURE TARGET GOES DOWN FOR REBOOT
sleep 60s

## Loop and wait for the Controller node to get discovered
NODECOUNT=2
count=1
while [ $count -lt 20 ]; do
    count=$(( count + 1 ))
    sleep 60s
    ELEMENTS=$(ssh ${SSH_OPTS} crowbar@${HEADNODE} 'sudo -i /opt/dell/bin/crowbar_node_state status --no-ready | grep "^d" | wc -l')
    if [ ${ELEMENTS} == ${NODECOUNT} ]; then
        log "All nodes discovered"
        break
    fi
    if [ $count -eq 20 ]; then
        log "ALL NODES NOT DISCOVERED!!"
        ssh ${SSH_OPTS} crowbar@${HEADNODE} 'sudo -i /opt/dell/bin/crowbar_node_state status --no-ready'
        exit 1
    fi
done
sleep 5

## Ghetto Hack since we are disabling IPMI Barclamp
## SET PXE BOOT on BOTH NODES 
ipmi_set_pxe ${CONTROLLER_DRAC_IP}
ipmi_set_pxe ${COMPUTE_DRAC_IP}

##################################################
# Push MYSQL Proposal
crowbar_proposal "mysql" "create"
crowbar_proposal "mysql" "commit"
crowbar_proposal_status "mysql" 30
##################################################

##################################################
# Push the Keystone Proposal
crowbar_proposal "keystone" "create"
crowbar_proposal "keystone" "commit"
crowbar_proposal_status "keystone"
##################################################

##################################################
# Push the Glance Proposal
crowbar_proposal "glance" "create"
crowbar_proposal "glance" "commit"
crowbar_proposal_status "glance"
##################################################

##################################################
# Push the Nova Proposal
crowbar_proposal "nova" "create"
crowbar_proposal "nova" "commit"
crowbar_proposal_status "nova" 30
##################################################

##################################################
# Push the Dash Proposal
crowbar_proposal "nova_dashboard" "create"
crowbar_proposal "nova_dashboard" "commit"
crowbar_proposal_status "nova_dashboard"
##################################################