From 1166eb16a47d98bc6b198a764c7a2cbc92c90d08 Mon Sep 17 00:00:00 2001 From: Ivan Remizov Date: Thu, 4 May 2017 23:04:00 +0300 Subject: [PATCH] Add template for compare xml job This patch adds template and couple of useful macro for performing xml comparisements of JJ changes --- library/macros/.gitkeep | 0 library/macros/stash-trigger.yaml | 94 ++++++++++++++++ library/scripts/.gitkeep | 0 library/scripts/jjb_compare_xml.sh | 127 ++++++++++++++++++++++ library/templates/.gitkeep | 0 library/templates/compare-xml.yaml | 67 ++++++++++++ servers/example/projects/.gitkeep | 0 servers/example/projects/compare-xml.yaml | 18 +++ 8 files changed, 306 insertions(+) delete mode 100644 library/macros/.gitkeep create mode 100644 library/macros/stash-trigger.yaml delete mode 100644 library/scripts/.gitkeep create mode 100644 library/scripts/jjb_compare_xml.sh delete mode 100644 library/templates/.gitkeep create mode 100644 library/templates/compare-xml.yaml delete mode 100644 servers/example/projects/.gitkeep create mode 100644 servers/example/projects/compare-xml.yaml diff --git a/library/macros/.gitkeep b/library/macros/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/library/macros/stash-trigger.yaml b/library/macros/stash-trigger.yaml new file mode 100644 index 0000000..154506d --- /dev/null +++ b/library/macros/stash-trigger.yaml @@ -0,0 +1,94 @@ +- parameter: + name: library/stash-trigger/parameters + + parameters: + # PR Info + - string: + name: pullRequestTitle + description: 'Pull Request: Title' + default: '' + - string: + name: pullRequestId + description: 'Pull Request: Identifier' + default: '' + # PR Source + - string: + name: projectCode + description: 'Source: Project' + default: '' + - string: + name: repositoryName + description: 'Source: Repository' + default: '' + - string: + name: sourceBranch + description: 'Source: Branch' + default: '' + - string: + name: sourceCommitHash + description: 'Source: Commit hash of Pull Request HEAD' + default: '' + # PR Target + - string: + name: destinationRepositoryOwner + description: 'Target: Project' + default: '' + - string: + name: destinationRepositoryName + description: 'Target: Repository' + default: '' + - string: + name: targetBranch + description: 'Target: Branch' + default: '' + + +- trigger: + name: library/stash-trigger/trigger + + triggers: + - stash: + spec: '{timer}' + cron: '{timer}' + stash_host: 'https://{host}/' + project_code: '{project-key}' + repository_name: '{repo-slug}' + credentials-id: '{credentials-id}' + username: '{username}' + password: '{password}' + ci_skip_phrases: 'NO TEST' + ci_build_phrases: 'test this please' + target_branches_to_build: '{branch}' + ignore_ssl: 'true' + check_destination_commit: 'false' + check_mergeable: 'false' + merge_on_success: 'false' + check_not_conflicted: 'false' + only_build_on_comment: 'false' + delete_previous_build_finish_comments: 'false' + cancel_outdated_jobs_enabled: 'true' + +- scm: + name: library/stash-trigger/scm + + scm: + - git: + url: 'ssh://git@{host}:{port}/$destinationRepositoryOwner/$destinationRepositoryName.git' + refspec: '+refs/pull-requests/*:refs/remotes/origin/pr/*' + credentials-id: '{credentials-id}' + branches: + - '$sourceCommitHash' + wipe-workspace: true # workaround for force-push and rebase + +- publisher: + name: library/stash-trigger/publish-feedback + + publishers: + - stash: + url: 'https://{host}' + username: '{username}' + password: '{password}' + credentials-id: '{credentials-id}' + ignore-ssl: false + commit-sha1: $GIT_COMMIT + include-build-number: true diff --git a/library/scripts/.gitkeep b/library/scripts/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/library/scripts/jjb_compare_xml.sh b/library/scripts/jjb_compare_xml.sh new file mode 100644 index 0000000..909953e --- /dev/null +++ b/library/scripts/jjb_compare_xml.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +set -ex + +# workaround for old RHELs, we need to install correct tox system-wide +rm -rf venv +virtualenv venv +source venv/bin/activate +pip install pip --upgrade +pip install tox +# --- + +JOBS_OUT_DIR=${WORKSPACE}/output/jobs +JOBS_LOGFILE=${JOBS_OUT_DIR}/jobs-diff.log +RESULT='' +BLOCKLIST=blocklist + +# First generate output from BASE_COMMIT vars value +git checkout "${targetBranch}" + +rm -rf .tox +for ENV in ${WORKSPACE}/servers/*; do + tox -e "${ENV##*/}" + mkdir -p "./output/jobs/old" + mv "./output/${ENV##*/}" "./output/jobs/old/${ENV##*/}" +done + +# Then use that as a reference to compare against HEAD +git checkout "origin/pr/${pullRequestId}/merge" +rm -rf .tox +for ENV in ${WORKSPACE}/servers/*; do + tox -e "${ENV##*/}" + mkdir -p "./output/jobs/new" + mv "./output/${ENV##*/}" "./output/jobs/new/${ENV##*/}" +done + +compare_xml() { + +# Replace arguments with built-in variables ($1 - path to jobs or views output directory, $2 - path to jobs or views log file) +OUT_DIR=$1 +LOGFILE=$2 +# Specifying for http links type of comaprison (jobs or views) +TYPE=$3 + +BLOCK=0 +CHANGE=0 +ADD=0 +REMOVE=0 + +BLOCKED="[blocked]
" +CHANGED="[changed]
" +ADDED="[added]
" +REMOVED="[removed]
" + +DIFF=$(diff -q -r -u "${OUT_DIR}/old" "${OUT_DIR}/new" &>"${LOGFILE}"; echo "${?}") +# Any changed job discovered? If exit code was 1, then there is a difference +if [[ ${DIFF} -eq 1 ]]; then + # Loop through all changed jobs and compare them with a blocklist + for JOB in $(awk '/Files/ {print $2}' "${LOGFILE}"); do + # Extract job's name + JOB_NAME=$(basename "${JOB}") + # Extract job's ENV name (server/${ENV} to make sure, + # that we are comparing ENV/JOB_NAME with right ENV/BLOCKLIST. + JOB_ENV=$(echo "${JOB}" | awk -F "/" '{print $(NF?NF-1:0)}') + # Make diff + mkdir -p "${OUT_DIR}/diff/${JOB_ENV}" + diff -U 50 "${OUT_DIR}/old/${JOB_ENV}/${JOB_NAME}" \ + "${OUT_DIR}/new/${JOB_ENV}/${JOB_NAME}" >> "${OUT_DIR}/diff/${JOB_ENV}/${JOB_NAME}" || true + + # fixme: add blocklists + # for BL in ${WORKSPACE}/servers/${JOB_ENV}/${BLOCKLIST}; do + # # Do exact job name match when checking with blocklist. + # GREP=$(grep -Fxq "${JOB_NAME}" "${BL}"; echo "${?}") + # if [[ ${GREP} -eq 0 ]]; then + # BLOCK=1 + # BLOCKED+=${JOB_ENV}/${JOB_NAME}\ + # # If grep returned 2 then there was no such blockfile. + # elif [[ ${GREP} -eq 2 ]]; then + # echo Error. There is no such blockfile. + # exit 2 + # else + CHANGE=1 + CHANGED+="${JOB_ENV}/${JOB_NAME}
" + # fi + # done + done + # Now find added/removed Jobs... + for JOB in $(awk '/Only in/ {print $3$4}' "${LOGFILE}"); do + ON=$(echo "${JOB}"|awk -F/ '{print $8}') + JOB_NAME=$(echo "${JOB}"| awk -F: '{print $2}') + JOB_ENV=$(echo "${JOB}" | awk -F "/" '{print $(NF?NF-0:0)}' | cut -f1 -d ':') + if [[ ${ON} = 'old' ]]; then + REMOVE=1 + REMOVED+="${JOB_ENV}/${JOB_NAME}
" + elif [[ ${ON} = 'new' ]]; then + ADD=1 + ADDED+="${JOB_ENV}/${JOB_NAME}
" + fi + done +fi + +# Add section only if there're any changes found +if [ "$(( BLOCK + CHANGE + ADD + REMOVE ))" -gt 0 ]; then + RESULT+="
$(tr "[:lower:]" "[:upper:]" <<< "${TYPE}"):
" +fi + +# Print Blocked or Changed jobs. +if [[ ${BLOCK} -eq 1 ]]; then + RESULT+=${BLOCKED} +elif [[ ${CHANGE} -eq 1 ]]; then + RESULT+=${CHANGED} +fi +# And print added/removed if any. +if [[ ${REMOVE} -eq 1 ]]; then + RESULT+=${REMOVED} +fi +if [[ ${ADD} -eq 1 ]]; then + RESULT+=${ADDED} +fi +} + +compare_xml "${JOBS_OUT_DIR}" "${JOBS_LOGFILE}" "jobs" +BLOCK_JOBS=${BLOCK} + +echo "${RESULT#"
"}" + +exit "$(( BLOCK_JOBS ))" diff --git a/library/templates/.gitkeep b/library/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/library/templates/compare-xml.yaml b/library/templates/compare-xml.yaml new file mode 100644 index 0000000..7401943 --- /dev/null +++ b/library/templates/compare-xml.yaml @@ -0,0 +1,67 @@ +- job-template: + id: 'library/compare-xml' + name: '{project-key}.{repo-slug}.{branch-display-name}.compare-xml' + + description: | +

Compare xmls +

+ Compare JJB output for PRs in {project-key}/{repo-slug} proposed to {branch} + + node: 'jjb_update' + + concurrent: true + + stash-poll-timer: '* * * * *' + + parameters: + - library/stash-trigger/parameters + + wrappers: + - inject-passwords: + global: true + mask-password-params: true + - ansicolor: + colormap: xterm + - timeout: + fail: true + timeout: '{timeout}' + write-description: true + + triggers: + - library/stash-trigger/trigger: + project-key: '{project-key}' + repo-slug: '{repo-slug}' + branch: '{branch}' + timer: '{stash-poll-timer}' + host: '{host}' + credentials-id: '{credentials-id}' + username: '{username}' + password: '{password}' + + scm: + - library/stash-trigger/scm: + host: '{host}' + port: '{port}' + username: '{username}' + password: '{password}' + credentials-id: '{credentials-id}' + + builders: + - shell: + !include-raw-escape: '../scripts/jjb_compare_xml.sh' + + publishers: + + - archive: + artifacts: 'output/**' + allow-empty: 'true' + + - description-setter: + regexp: (^(JOBS|VIEWS):
\[(?!blocked)[a-z]*\].*) + regexp-for-failed: (^(JOBS|VIEWS):
\[blocked\].*) + + - library/stash-trigger/publish-feedback: + host: '{host}' + username: '{username}' + password: '{password}' + credentials-id: '{credentials-id}' \ No newline at end of file diff --git a/servers/example/projects/.gitkeep b/servers/example/projects/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/servers/example/projects/compare-xml.yaml b/servers/example/projects/compare-xml.yaml new file mode 100644 index 0000000..83c7f80 --- /dev/null +++ b/servers/example/projects/compare-xml.yaml @@ -0,0 +1,18 @@ +- project: + name: compare-xml + + timeout: 10 + + project-key: AA # example + repo-slug: jjb-library # example + branch-display-name: master + branch: master + + host: git.my-bitbucket.org + port: 443 + credentials-id: '' + username: '' + password: '' + + jobs: + - library/compare-xml \ No newline at end of file