Add template for compare xml job

This patch adds template and couple of useful macro
for performing xml comparisements of JJ changes

library/macros/stash-trigger.yaml

@@ -0,0 +1,94 @@
+- parameter:
+    name: library/stash-trigger/parameters
+
+    parameters:
+    # PR Info
+    - string:
+        name: pullRequestTitle
+        description: 'Pull Request: Title'
+        default: ''
+    - string:
+        name: pullRequestId
+        description: 'Pull Request: Identifier'
+        default: ''
+    # PR Source
+    - string:
+        name: projectCode
+        description: 'Source: Project'
+        default: ''
+    - string:
+        name: repositoryName
+        description: 'Source: Repository'
+        default: ''
+    - string:
+        name: sourceBranch
+        description: 'Source: Branch'
+        default: ''
+    - string:
+        name: sourceCommitHash
+        description: 'Source: Commit hash of Pull Request HEAD'
+        default: ''
+    # PR Target
+    - string:
+        name: destinationRepositoryOwner
+        description: 'Target: Project'
+        default: ''
+    - string:
+        name: destinationRepositoryName
+        description: 'Target: Repository'
+        default: ''
+    - string:
+        name: targetBranch
+        description: 'Target: Branch'
+        default: ''
+
+
+- trigger:
+    name: library/stash-trigger/trigger
+
+    triggers:
+    - stash:
+        spec: '{timer}'
+        cron: '{timer}'
+        stash_host: 'https://{host}/'
+        project_code: '{project-key}'
+        repository_name: '{repo-slug}'
+        credentials-id: '{credentials-id}'
+        username: '{username}'
+        password: '{password}'
+        ci_skip_phrases: 'NO TEST'
+        ci_build_phrases: 'test this please'
+        target_branches_to_build: '{branch}'
+        ignore_ssl: 'true'
+        check_destination_commit: 'false'
+        check_mergeable: 'false'
+        merge_on_success: 'false'
+        check_not_conflicted: 'false'
+        only_build_on_comment: 'false'
+        delete_previous_build_finish_comments: 'false'
+        cancel_outdated_jobs_enabled: 'true'
+
+- scm:
+    name: library/stash-trigger/scm
+
+    scm:
+    - git:
+        url: 'ssh://git@{host}:{port}/$destinationRepositoryOwner/$destinationRepositoryName.git'
+        refspec: '+refs/pull-requests/*:refs/remotes/origin/pr/*'
+        credentials-id: '{credentials-id}'
+        branches:
+        - '$sourceCommitHash'
+        wipe-workspace: true  # workaround for force-push and rebase
+
+- publisher:
+    name: library/stash-trigger/publish-feedback
+
+    publishers:
+    - stash:
+        url: 'https://{host}'
+        username: '{username}'
+        password: '{password}'
+        credentials-id: '{credentials-id}'
+        ignore-ssl: false
+        commit-sha1: $GIT_COMMIT
+        include-build-number: true

library/scripts/jjb_compare_xml.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+
+set -ex
+
+# workaround for old RHELs, we need to install correct tox system-wide
+rm -rf venv
+virtualenv venv
+source venv/bin/activate
+pip install pip --upgrade
+pip install tox
+# ---
+
+JOBS_OUT_DIR=${WORKSPACE}/output/jobs
+JOBS_LOGFILE=${JOBS_OUT_DIR}/jobs-diff.log
+RESULT=''
+BLOCKLIST=blocklist
+
+# First generate output from BASE_COMMIT vars value
+git checkout "${targetBranch}"
+
+rm -rf .tox
+for ENV in ${WORKSPACE}/servers/*; do
+  tox -e "${ENV##*/}"
+  mkdir -p "./output/jobs/old"
+  mv "./output/${ENV##*/}" "./output/jobs/old/${ENV##*/}"
+done
+
+# Then use that as a reference to compare against HEAD
+git checkout "origin/pr/${pullRequestId}/merge"
+rm -rf .tox
+for ENV in ${WORKSPACE}/servers/*; do
+  tox -e "${ENV##*/}"
+  mkdir -p "./output/jobs/new"
+  mv "./output/${ENV##*/}" "./output/jobs/new/${ENV##*/}"
+done
+
+compare_xml() {
+
+# Replace arguments with built-in variables ($1 - path to jobs or views output directory, $2 - path to jobs or views log file)
+OUT_DIR=$1
+LOGFILE=$2
+# Specifying for http links type of comaprison (jobs or views)
+TYPE=$3
+
+BLOCK=0
+CHANGE=0
+ADD=0
+REMOVE=0
+
+BLOCKED="[blocked]<br>"
+CHANGED="[changed]<br>"
+ADDED="[added]<br>"
+REMOVED="[removed]<br>"
+
+DIFF=$(diff -q -r -u "${OUT_DIR}/old" "${OUT_DIR}/new" &>"${LOGFILE}"; echo "${?}")
+# Any changed job discovered? If exit code was 1, then there is a difference
+if [[ ${DIFF} -eq 1 ]]; then
+  # Loop through all changed jobs and compare them with a blocklist
+  for JOB in $(awk '/Files/ {print $2}' "${LOGFILE}"); do
+    # Extract job's name
+    JOB_NAME=$(basename "${JOB}")
+    # Extract job's ENV name (server/${ENV} to make sure,
+    # that we are comparing ENV/JOB_NAME with right ENV/BLOCKLIST.
+    JOB_ENV=$(echo "${JOB}" | awk -F "/" '{print $(NF?NF-1:0)}')
+    # Make diff
+    mkdir -p "${OUT_DIR}/diff/${JOB_ENV}"
+    diff -U 50 "${OUT_DIR}/old/${JOB_ENV}/${JOB_NAME}" \
+        "${OUT_DIR}/new/${JOB_ENV}/${JOB_NAME}" >> "${OUT_DIR}/diff/${JOB_ENV}/${JOB_NAME}" || true
+
+    # fixme: add blocklists
+    # for BL in ${WORKSPACE}/servers/${JOB_ENV}/${BLOCKLIST}; do
+      # # Do exact job name match when checking with blocklist.
+      # GREP=$(grep -Fxq "${JOB_NAME}" "${BL}"; echo "${?}")
+      # if [[ ${GREP} -eq 0 ]]; then
+      #   BLOCK=1
+      #   BLOCKED+=${JOB_ENV}/${JOB_NAME}\<br\>
+      # # If grep returned 2 then there was no such blockfile.
+      # elif [[ ${GREP} -eq 2 ]]; then
+      #   echo Error. There is no such blockfile.
+      #   exit 2
+      # else
+        CHANGE=1
+        CHANGED+="<a href=${BUILD_URL}artifact/output/${TYPE}/diff/${JOB_ENV}/${JOB_NAME}/*view*/>${JOB_ENV}/${JOB_NAME}</a><br>"
+      # fi
+    # done
+  done
+  # Now find added/removed Jobs...
+  for JOB in $(awk '/Only in/ {print $3$4}' "${LOGFILE}"); do
+    ON=$(echo  "${JOB}"|awk -F/ '{print $8}')
+    JOB_NAME=$(echo  "${JOB}"| awk -F: '{print $2}')
+    JOB_ENV=$(echo "${JOB}" | awk -F "/" '{print $(NF?NF-0:0)}' | cut -f1 -d ':')
+    if [[ ${ON} = 'old' ]]; then
+      REMOVE=1
+      REMOVED+="<a href=${BUILD_URL}artifact/output/${TYPE}/old/${JOB_ENV}/${JOB_NAME}/*view*/>${JOB_ENV}/${JOB_NAME}</a><br>"
+    elif [[ ${ON} = 'new' ]]; then
+      ADD=1
+      ADDED+="<a href=${BUILD_URL}artifact/output/${TYPE}/new/${JOB_ENV}/${JOB_NAME}/*view*/>${JOB_ENV}/${JOB_NAME}</a><br>"
+    fi
+  done
+fi
+
+# Add section only if there're any changes found
+if [ "$(( BLOCK + CHANGE + ADD + REMOVE ))" -gt 0 ]; then
+  RESULT+="<br><b>$(tr "[:lower:]" "[:upper:]" <<< "${TYPE}"):</b><br>"
+fi
+
+# Print Blocked or Changed jobs.
+if [[ ${BLOCK} -eq 1 ]]; then
+  RESULT+=${BLOCKED}
+elif [[ ${CHANGE} -eq 1 ]]; then
+  RESULT+=${CHANGED}
+fi
+# And print added/removed if any.
+if [[ ${REMOVE} -eq 1 ]]; then
+  RESULT+=${REMOVED}
+fi
+if [[ ${ADD} -eq 1 ]]; then
+  RESULT+=${ADDED}
+fi
+}
+
+compare_xml "${JOBS_OUT_DIR}" "${JOBS_LOGFILE}" "jobs"
+BLOCK_JOBS=${BLOCK}
+
+echo "${RESULT#"<br>"}"
+
+exit "$(( BLOCK_JOBS ))"

library/templates/compare-xml.yaml

@@ -0,0 +1,67 @@
+- job-template:
+    id: 'library/compare-xml'
+    name: '{project-key}.{repo-slug}.{branch-display-name}.compare-xml'
+
+    description: |
+      <h2> Compare xmls
+      <p>
+      Compare JJB output for PRs in {project-key}/{repo-slug} proposed to {branch}
+
+    node: 'jjb_update'
+
+    concurrent: true
+
+    stash-poll-timer: '* * * * *'
+
+    parameters:
+    - library/stash-trigger/parameters
+
+    wrappers:
+    - inject-passwords:
+        global: true
+        mask-password-params: true
+    - ansicolor:
+        colormap: xterm
+    - timeout:
+        fail: true
+        timeout: '{timeout}'
+        write-description: true
+
+    triggers:
+    - library/stash-trigger/trigger:
+        project-key: '{project-key}'
+        repo-slug: '{repo-slug}'
+        branch: '{branch}'
+        timer: '{stash-poll-timer}'
+        host: '{host}'
+        credentials-id: '{credentials-id}'
+        username: '{username}'
+        password: '{password}'
+
+    scm:
+    - library/stash-trigger/scm:
+        host: '{host}'
+        port: '{port}'
+        username: '{username}'
+        password: '{password}'
+        credentials-id: '{credentials-id}'
+
+    builders:
+    - shell:
+        !include-raw-escape: '../scripts/jjb_compare_xml.sh'
+
+    publishers:
+
+    - archive:
+        artifacts: 'output/**'
+        allow-empty: 'true'
+
+    - description-setter:
+        regexp: (^<b>(JOBS|VIEWS):</b><br>\[(?!blocked)[a-z]*\].*)
+        regexp-for-failed: (^<b>(JOBS|VIEWS):</b><br>\[blocked\].*)
+
+    - library/stash-trigger/publish-feedback:
+        host: '{host}'
+        username: '{username}'
+        password: '{password}'
+        credentials-id: '{credentials-id}'

servers/example/projects/compare-xml.yaml

@@ -0,0 +1,18 @@
+- project:
+    name: compare-xml
+
+    timeout: 10
+
+    project-key: AA  # example
+    repo-slug: jjb-library  # example
+    branch-display-name: master
+    branch: master
+
+    host: git.my-bitbucket.org
+    port: 443
+    credentials-id: ''
+    username: ''
+    password: ''
+
+    jobs:
+    - library/compare-xml