From 256b3d15a9ecce5192e3c6afadf63cce1ce03979 Mon Sep 17 00:00:00 2001 From: David Cantrell Date: Thu, 26 Mar 2020 15:42:26 -0400 Subject: [PATCH 1/4] Shell syntax and portability fixes for create-containers.sh This removes the use of bash-isms and simplifies some of the shell syntax constructs. I use ShellCheck as well to validate my changes. The resulting script runs under bash, but also shells like dash and zsh. --- config/s2i/create-containers.sh | 62 ++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/config/s2i/create-containers.sh b/config/s2i/create-containers.sh index cf75d31a..607e7048 100755 --- a/config/s2i/create-containers.sh +++ b/config/s2i/create-containers.sh @@ -1,4 +1,6 @@ -#!/bin/bash +#!/bin/sh + +PATH=/bin:/usr/bin # set to 1 to enable debugging DEBUG=0 @@ -10,37 +12,46 @@ project="continuous-infra" templates="fedoraci-runner/fedoraci-runner-buildconfig-template.yml \ jenkins/jenkins-fedoraci-slave-buildconfig-template.yaml" -function logerror { - echo "Error: $1" +logerror() { + echo "Error: $1" >&2 exit 1 } -function logwarning { - echo "Warning: $1" +logwarning() { + echo "Warning: $1" >&2 } -function logdebug { - if [ ${DEBUG} -eq 1 ] ; then - echo "DEBUG: $1" +logdebug() { + if [ "${DEBUG}" = "1" ] ; then + echo "DEBUG: $1" >&2 fi } -function loginfo { +loginfo() { echo "$1" } -function cleanup { - rm -rf ${ci_pipeline_location} +cleanup() { + [ -z "${ci_pipeline_location:=}" ] || rm -rf "${ci_pipeline_location}" } -function verifyEnv { +verifyEnv() { ## jq - command -v jq >/dev/null 2>&1 || { echo "Require jq but it's not installed. Aborting." >&2; exit 1; } + jq --help >/dev/null 2>&1 + if [ $? -eq 127 ]; then + echo "Require jq but it's not installed. Aborting." >&2 + exit 1 + fi + ## oc - command -v oc >/dev/null 2>&1 || { echo "Require oc but it's not installed. Aborting." >&2; exit 1; } + oc --help >/dev/null 2>&1 + if [ $? -eq 127 ]; then + echo "Require oc but it's not installed. Aborting." >&2 + exit 1 + fi } -function processTemplate { +processTemplate() { templateFile="${1}" loginfo "* Processing ${templateFile}..." templateName=$(oc process -f "${templateFile}" | jq '.items[1].metadata.labels.template' | sed 's/"//g') @@ -50,9 +61,7 @@ function processTemplate { buildConfigName=$(oc process -f "${templateFile}" | jq '.items[1].metadata.name' | sed 's/"//g') logdebug " - Build Config name is ${buildConfigName}" - oc get template "${templateName}" > /dev/null 2>&1 - - if [ $? -ne 0 ] ; then + if ! oc get template "${templateName}" > /dev/null 2>&1 ; then loginfo " >> Creating Build Config Template ${templateName}" oc create -f "${templateFile}" > /dev/null 2>&1 || { echo "Failed to create build config! Aborting." >&2; exit 1; } else @@ -61,21 +70,19 @@ function processTemplate { fi imageExists=0 - oc get imagestream "${imageStreamName}" > /dev/null 2>&1 - if [ $? -eq 0 ] ; then + if oc get imagestream "${imageStreamName}" > /dev/null 2>&1 ; then logdebug " Image Stream ${imageStreamName} already exists" imageExists=1 fi buildConfigExists=0 - oc get buildconfig "${buildConfigName}" > /dev/null 2>&1 - if [ $? -eq 0 ] ; then + if oc get buildconfig "${buildConfigName}" > /dev/null 2>&1 ; then logdebug " Build Config ${buildConfigName} already exists" buildConfigExists=1 fi - if [[ ${imageExists} -eq 0 ]] && [[ ${buildConfigExists} -eq 0 ]] ; then + if [ ${imageExists} -eq 0 ] && [ ${buildConfigExists} -eq 0 ] ; then loginfo " >> Image Stream and Build Config do not exist. Creating..." - oc new-app "${templateName}" ${REPO_URL_PARAM} ${REPO_REF_PARAM} > /dev/null 2>&1 || { echo "Failed to create new app! Aborting." >&2; exit 1; } + oc new-app "${templateName}" "${REPO_URL_PARAM}" "${REPO_REF_PARAM}" > /dev/null 2>&1 || { echo "Failed to create new app! Aborting." >&2; exit 1; } fi loginfo "" } @@ -97,16 +104,15 @@ else REPO_REF_PARAM="-p REPO_REF=${REPO_REF}" fi -oc project "${project}" > /dev/null 2>&1 -if [ $? -ne 0 ] ; then +if ! oc project "${project}" > /dev/null 2>&1 ; then logdebug "Project does not exist...Creating..." oc new-project "${project}" > /dev/null 2>&1 || { echo "Failed to create new project! Aborting." >&2; exit 1; } oc project "${project}" > /dev/null 2>&1 fi -trap cleanup EXIT SIGHUP SIGINT SIGTERM +trap cleanup EXIT HUP INT TERM -for template in ${templates[@]}; do +for template in ${templates}; do processTemplate "${template}" done From 4e9941b378d610ed92243e685e81b75cc1ec5eb4 Mon Sep 17 00:00:00 2001 From: David Cantrell Date: Thu, 26 Mar 2020 16:18:43 -0400 Subject: [PATCH 2/4] Shell syntax and portability fixes for koji_build_pr.sh --- .../fedoraci-runner/rpmbuild/koji_build_pr.sh | 135 +++++++++++------- 1 file changed, 81 insertions(+), 54 deletions(-) diff --git a/config/Dockerfiles/fedoraci-runner/rpmbuild/koji_build_pr.sh b/config/Dockerfiles/fedoraci-runner/rpmbuild/koji_build_pr.sh index 498cf7c3..9bca6a55 100755 --- a/config/Dockerfiles/fedoraci-runner/rpmbuild/koji_build_pr.sh +++ b/config/Dockerfiles/fedoraci-runner/rpmbuild/koji_build_pr.sh @@ -1,69 +1,96 @@ -#!/bin/bash +#!/bin/sh # This container builds with koji into $RPMDIR starting from a pagure PR -set -xeo pipefail -# avoid using head -n 1 as with pipefail it might cause the command to exit with error 141 due to broken pipe - +PATH=/bin:/usr/bin # Check to make sure we have all required vars -if [ -z "${fed_repo}" ]; then echo "No fed_repo env var" ; exit 1 ; fi -if [ -z "${fed_id}" ]; then echo "No fed_id env var" ; exit 1 ; fi -if [ -z "${fed_uid}" ]; then echo "No fed_uid env var" ; exit 1 ; fi -if [ -z "${FEDORA_PRINCIPAL}" ]; then echo "No FEDORA_PRINCIPAL env var"; exit 1; fi -if [ -z "${PAGURE_URL}" ]; then echo "No PAGURE_URL env var"; exit 1; fi - -CURRENTDIR=$(pwd) -if [ ${CURRENTDIR} == "/" ] ; then - cd /home +if [ -z "${fed_repo:=}" ]; then + echo "No fed_repo env var" >&2 + exit 1 +fi + +if [ -z "${fed_id:=}" ]; then + echo "No fed_id env var" >&2 + exit 1 +fi + +if [ -z "${fed_uid:=}" ]; then + echo "No fed_uid env var" >&2 + exit 1 +fi + +if [ -z "${FEDORA_PRINCIPAL:=}" ]; then + echo "No FEDORA_PRINCIPAL env var" >&2 + exit 1 +fi + +if [ -z "${PAGURE_URL:=}" ]; then + echo "No PAGURE_URL env var" >&2 + exit 1 +fi + +CURRENTDIR="$(realpath "$(pwd)")" + +if [ "${CURRENTDIR}" = "/" ] ; then + if [ ! -d /home ]; then + echo "Missing /home" >&2 + exit 1 + fi + + cd /home || exit 1 CURRENTDIR=/home fi # Allow change koji server to be used -KOJI_PARAMS=${KOJI_PARAMS:-} +KOJI_PARAMS="${KOJI_PARAMS:-}" -RPMDIR=${CURRENTDIR}/${fed_repo}_repo +RPMDIR="${CURRENTDIR}/${fed_repo}_repo" # Create one dir to store logs in that will be mounted -LOGDIR=${CURRENTDIR}/logs -rm -rf ${LOGDIR}/* -mkdir -p ${LOGDIR} +LOGDIR="${CURRENTDIR}/logs" +[ -d "${LOGDIR}" ] && rm -rf "${LOGDIR:?}"/* +mkdir -p "${LOGDIR}" # Clone the fedoraproject git repo -rm -rf ${fed_repo} -git clone ${PAGURE_URL}/${fed_namespace}/${fed_repo}.git -if [ "$?" != 0 ]; then echo -e "ERROR: GIT CLONE\nSTATUS: $?"; exit 1; fi -pushd ${fed_repo} +rm -rf "${fed_repo}" + +if ! git clone "${PAGURE_URL}"/"${fed_namespace:?}"/"${fed_repo}".git ; then + retcode=$? + echo "ERROR: GIT CLONE" + echo "STATUS: ${retcode}" + exit 1 +fi + +cd "${fed_repo}" || exit 1 # Checkout the branch and apply the patch to HEAD of branch -git checkout ${fed_branch} -git fetch -fu origin refs/pull/${fed_id}/head:pr +git checkout "${fed_branch:?}" +git fetch -fu origin refs/pull/"${fed_id}"/head:pr # Setting git config and merge message in case we try to merge a closed PR, like it is done on stage instance git -c "user.name=Fedora CI" -c "user.email=ci@lists.fedoraproject.org" merge pr -m "Fedora CI pipeline" # Get current NVR -truenvr=$(rpm -q --define "dist .$DIST_BRANCH" --queryformat '%{name}-%{version}-%{release}\n' --specfile ${fed_repo}.spec | awk 'NR==1') +truenvr=$(rpm -q --define "dist .$DIST_BRANCH" --queryformat '%{name}-%{version}-%{release}\n' --specfile "${fed_repo}".spec | awk 'NR==1') echo "original_spec_nvr=${truenvr}" >> ${LOGDIR}/job.props -# Find number of git commits in log to append to RELEASE before %{?dist} -commits=$(git log --pretty=format:'' | wc -l) # Build srpm to send to koji -fedpkg --release ${fed_branch} srpm -VERSION=$(rpmspec --queryformat "%{VERSION}\n" -q ${fed_repo}.spec | awk 'NR==1') +fedpkg --release "${fed_branch}" srpm +VERSION=$(rpmspec --queryformat "%{VERSION}\n" -q "${fed_repo}".spec | awk 'NR==1') # Set up koji creds -kinit -k -t "${CURRENTDIR}/fedora.keytab" $FEDORA_PRINCIPAL +kinit -k -t "${CURRENTDIR}/fedora.keytab" "${FEDORA_PRINCIPAL}" # Some packages are requiring configure not be run as root, so set this to bypass the error export FORCE_UNSAFE_CONFIGURE=1 # Build the package with koji # ignore koij exit status. If build fails it will be detected later, https://pagure.io/fedora-ci/general/issue/76 -koji ${KOJI_PARAMS} build --wait --arch-override=x86_64 --scratch ${branch} ${fed_repo}*.src.rpm | tee ${LOGDIR}/kojioutput.txt || true +koji "${KOJI_PARAMS}" build --wait --arch-override=x86_64 --scratch "${fed_branch}" "${fed_repo}"*.src.rpm | tee ${LOGDIR}/kojioutput.txt || true -popd +cd "${CURRENTDIR}" || exit 1 -SCRATCHID=$(cat ${LOGDIR}/kojioutput.txt | awk '/Created task:/ { print $3 }') -if ! [[ $SCRATCHID =~ ^[0-9]+$ ]]; then +SCRATCHID=$(awk '/Created task:/ { print $3 }' < ${LOGDIR}/kojioutput.txt) +if ! echo "${SCRATCHID}" | grep -q -E '^[0-9]+$' >/dev/null 2>&1 ; then echo "status=FAIL" >> ${LOGDIR}/job.props - echo -e "ERROR: KOJI BUILD" + echo "ERROR: KOJI BUILD" exit 1 fi @@ -73,50 +100,50 @@ echo "koji_task_id=${SCRATCHID}" >> ${LOGDIR}/job.props TASK_STATE="unknown" while echo ${TASK_STATE} | grep -Ev "closed|failed|cancelled"; do # Wait for build to finish as the command can exit before build finishes, ignore exit code - koji watch-task ${SCRATCHID} || true - for i in {1..5}; do - if koji taskinfo ${SCRATCHID} | tee ${LOGDIR}/taskinfo.txt; then + koji watch-task "${SCRATCHID}" || true + for i in $(seq 1 5); do + if koji taskinfo "${SCRATCHID}" | tee ${LOGDIR}/taskinfo.txt; then break fi - if [[ $i -eq 5 ]]; then + if [ "${i}" -eq 5 ]; then echo "status=FAIL" >> ${LOGDIR}/job.props - echo -e "ERROR: KOJI TASK_INFO" + echo "ERROR: KOJI TASK_INFO" exit 1 fi sleep 60 done - TASK_STATE=$(cat ${LOGDIR}/taskinfo.txt | grep "State:" | awk '{print$2}') + TASK_STATE=$(grep "State:" ${LOGDIR}/taskinfo.txt | awk '{print$2}') done -if [ ${TASK_STATE} != "closed" ]; then +if [ ! "${TASK_STATE}" = "closed" ]; then echo "status=FAIL" >> ${LOGDIR}/job.props - echo -e "ERROR: KOJI BUILD\nSTATUS: $TASK_STATE" + echo "ERROR: KOJI BUILD" + echo "STATUS: ${TASK_STATE}" exit 1 fi echo "status=SUCCESS" >> ${LOGDIR}/job.props # Make repo to download rpms to -rm -rf ${RPMDIR} -mkdir -p ${RPMDIR} +rm -rf "${RPMDIR}" +mkdir -p "${RPMDIR}" # Create repo -pushd ${RPMDIR} -for i in {1..5}; do - koji ${KOJI_PARAMS} download-build --arch=x86_64 --arch=src --arch=noarch --debuginfo --task-id ${SCRATCHID} || koji ${KOJI_PARAMS} download-task --arch=x86_64 --arch=src --arch=noarch --logs ${SCRATCHID} && break - echo "koji build download failed, attempt: $i/5" - if [[ $i -lt 5 ]]; then +cd "${RPMDIR}" || exit 1 +for i in $(seq 1 5); do + koji "${KOJI_PARAMS}" download-build --arch=x86_64 --arch=src --arch=noarch --debuginfo --task-id "${SCRATCHID}" || koji "${KOJI_PARAMS}" download-task --arch=x86_64 --arch=src --arch=noarch --logs "${SCRATCHID}" && break + echo "koji build download failed, attempt: ${i}/5" + if [ "${i}" -lt 5 ]; then sleep 10 else exit 1 fi done createrepo . -popd +cd "${CURRENTDIR}" || exit 1 # Store modified nvr as well set +e -RPM_TO_CHECK=$(find ${RPMDIR}/ -name "${fed_repo}-${VERSION}*" | awk 'NR==1') -RPM_NAME=$(basename $RPM_TO_CHECK) -NVR=$(rpm --queryformat "%{NAME}-%{VERSION}-%{RELEASE}\n" -qp $RPM_TO_CHECK) +RPM_TO_CHECK=$(find "${RPMDIR}"/ -name "${fed_repo}-${VERSION}*" | awk 'NR==1') +NVR=$(rpm --queryformat "%{NAME}-%{VERSION}-%{RELEASE}\n" -qp "${RPM_TO_CHECK}") echo "nvr=${NVR}" >> ${LOGDIR}/job.props exit 0 From 1f37a63d7d3f37b785dc2ea2881f189411d88954 Mon Sep 17 00:00:00 2001 From: David Cantrell Date: Thu, 26 Mar 2020 16:24:51 -0400 Subject: [PATCH 3/4] Shell syntax and portability fixes for pull_old_task.sh --- .../fedoraci-runner/rpmbuild/pull_old_task.sh | 59 +++++++++++-------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/config/Dockerfiles/fedoraci-runner/rpmbuild/pull_old_task.sh b/config/Dockerfiles/fedoraci-runner/rpmbuild/pull_old_task.sh index b24b7514..225475be 100755 --- a/config/Dockerfiles/fedoraci-runner/rpmbuild/pull_old_task.sh +++ b/config/Dockerfiles/fedoraci-runner/rpmbuild/pull_old_task.sh @@ -1,25 +1,32 @@ -#!/bin/bash +#!/bin/sh set -xe +PATH=/bin:/usr/bin # Ensure we have required variable if [ -z "${PROVIDED_KOJI_TASKID}" ]; then echo "No task id variable provided" ; exit 1 ; fi -CURRENTDIR=$(pwd) -if [ ${CURRENTDIR} == "/" ] ; then - cd /home +CURRENTDIR="$(realpath "$(pwd)")" + +if [ "${CURRENTDIR}" = "/" ] ; then + if [ ! -d /home ]; then + echo "Missing /home" >&2 + exit 1 + fi + + cd /home || exit 1 CURRENTDIR=/home fi -LOGDIR=${CURRENTDIR}/logs -rm -rf ${LOGDIR}/* -mkdir ${LOGDIR} +LOGDIR="${CURRENTDIR}/logs" +[ -d "${LOGDIR}" ] && rm -rf "${LOGDIR:?}"/* +mkdir -p "${LOGDIR}" # Allow change koji server to be used KOJI_PARAMS=${KOJI_PARAMS:-} # Create trap function to archive as many of the variables as we have defined -function archive_variables { +archive_variables() { set +e cat << EOF > ${LOGDIR}/job.props koji_task_id=${PROVIDED_KOJI_TASKID} @@ -31,50 +38,50 @@ rpm_repo=${RPMDIR} EOF rm -rf somewhere } -trap archive_variables EXIT SIGHUP SIGINT SIGTERM +trap archive_variables EXIT HUP INT TERM mkdir somewhere -pushd somewhere +cd somewhere # Download koji build so we can archive it -for i in {1..5}; do - koji ${KOJI_PARAMS} download-build --arch=x86_64 --arch=src --arch=noarch --debuginfo --task-id ${PROVIDED_KOJI_TASKID} || koji ${KOJI_PARAMS} download-task --arch=x86_64 --arch=src --arch=noarch --logs ${PROVIDED_KOJI_TASKID} && break - echo "koji build download failed, attempt: $i/5" - if [[ $i -lt 5 ]]; then +for i in $(seq 1 5); do + koji "${KOJI_PARAMS}" download-build --arch=x86_64 --arch=src --arch=noarch --debuginfo --task-id "${PROVIDED_KOJI_TASKID}" || koji "${KOJI_PARAMS}" download-task --arch=x86_64 --arch=src --arch=noarch --logs "${PROVIDED_KOJI_TASKID}" && break + echo "koji build download failed, attempt: ${i}/5" + if [ "${i}" -lt 5 ]; then sleep 10 else exit 1 fi done createrepo . -PACKAGE=$(rpm --queryformat "%{NAME}\n" -qp *.src.rpm) -NVR=$(rpm --queryformat "%{NAME}-%{VERSION}-%{RELEASE}\n" -qp *.src.rpm) -popd +PACKAGE=$(rpm --queryformat "%{NAME}\n" -qp ./*.src.rpm) +NVR=$(rpm --queryformat "%{NAME}-%{VERSION}-%{RELEASE}\n" -qp ./*.src.rpm) +cd "${CURRENTDIR}" # check if need to create extra repo for group build if [ -n "${ADDITIONAL_TASK_IDS:-}" ]; then mkdir ${CURRENTDIR}/additional_tasks_repo - pushd ${CURRENTDIR}/additional_tasks_repo + cd ${CURRENTDIR}/additional_tasks_repo for additional_taskid in ${ADDITIONAL_TASK_IDS}; do # Download koji build so we can archive it - for i in {1..5}; do - koji download-build --noprogress --arch=x86_64 --arch=src --arch=noarch --debuginfo --task-id ${additional_taskid} || koji download-task --arch=x86_64 --arch=src --arch=noarch --logs ${additional_taskid} && break + for i in $(seq 1 5); do + koji download-build --noprogress --arch=x86_64 --arch=src --arch=noarch --debuginfo --task-id "${additional_taskid}" || koji download-task --arch=x86_64 --arch=src --arch=noarch --logs "${additional_taskid}" && break echo "koji additional task download failed, attempt: $i/5" - if [[ $i -lt 5 ]]; then + if [ "${i}" -lt 5 ]; then sleep 10 else - echo "koji additional task download failed!" + echo "koji additional task download failed!" >&2 exit 1 fi done done createrepo . - popd + cd "${CURRENTDIR}" fi RPMDIR=${CURRENTDIR}/${PACKAGE}_repo -rm -rf ${RPMDIR} -mkdir -p ${RPMDIR} +rm -rf "${RPMDIR}" +mkdir -p "${RPMDIR}" -mv somewhere/* ${RPMDIR}/ +mv somewhere/* "${RPMDIR}"/ archive_variables From 026a19d366c86cd21e5c23b737227b12c0c16f7c Mon Sep 17 00:00:00 2001 From: David Cantrell Date: Thu, 26 Mar 2020 16:26:44 -0400 Subject: [PATCH 4/4] Shell syntax and portability fixes for repoquery.sh --- .../fedoraci-runner/rpmbuild/repoquery.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/config/Dockerfiles/fedoraci-runner/rpmbuild/repoquery.sh b/config/Dockerfiles/fedoraci-runner/rpmbuild/repoquery.sh index a413b0e1..b011e42d 100755 --- a/config/Dockerfiles/fedoraci-runner/rpmbuild/repoquery.sh +++ b/config/Dockerfiles/fedoraci-runner/rpmbuild/repoquery.sh @@ -1,24 +1,25 @@ -#!/bin/bash +#!/bin/sh # This script ensures that $rpm_repo actually # contains an installable $fed_repo rpm set -xe +PATH=/bin:/usr/bin -if [[ -z "${fed_repo}" ]]; then - echo "This container requires fed_repo to be defined. Exiting..." +if [ -z "${fed_repo:=}" ]; then + echo "This container requires fed_repo to be defined. Exiting..." >&2 exit 1 fi -if [[ -z "${rpm_repo}" ]]; then +if [ -z "${rpm_repo}" ]; then rpm_repo=$(pwd)/${fed_repo}_repo fi # Get output to see if any rpms exist -output=$(dnf repoquery --disablerepo=\* --enablerepo=${fed_repo} --repofrompath=${fed_repo},${rpm_repo} --nvr ${fed_repo}) +output=$(dnf repoquery --disablerepo=\* --enablerepo="${fed_repo}" --repofrompath="${fed_repo}","${rpm_repo}" --nvr "${fed_repo}") -if [ ${output} == "" ]; then - echo "No installable rpms found! Failing" +if [ -z "${output}" ]; then + echo "No installable rpms found! Failing" >&2 exit 1 else exit 0