From 63d1e1e4f497465820ecf2d80e0810c42dc9b828 Mon Sep 17 00:00:00 2001 From: Niklas Date: Fri, 8 Sep 2023 16:15:15 +0200 Subject: [PATCH 01/17] #143 Add re release target for dogus (wip) - retreive remote and local cves (wip) --- build/make/re-release.sh | 61 ++++++++++++++++ build/make/release.mk | 6 +- build/make/release_functions.sh | 125 ++++++++++++++++++++++++++------ 3 files changed, 167 insertions(+), 25 deletions(-) create mode 100755 build/make/re-release.sh diff --git a/build/make/re-release.sh b/build/make/re-release.sh new file mode 100755 index 0000000..7d7b9bf --- /dev/null +++ b/build/make/re-release.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# Extension points in release.sh: +# +# A custom release argument file will be sourced if found. The custom release arg file may implement one or more bash +# functions which either release.sh or release_functions.sh define. If such a custom release function is found the +# release script must define the argument list which the custom release function will receive during the release. + +#sourceCustomReleaseArgs() { +# RELEASE_ARGS_FILE="${1}" +# +# if [[ -f "${RELEASE_ARGS_FILE}" ]]; then +# echo "Using custom release args file ${RELEASE_ARGS_FILE}" +# +# sourceCustomReleaseExitCode=0 +# # shellcheck disable=SC1090 +# source "${RELEASE_ARGS_FILE}" || sourceCustomReleaseExitCode=$? +# if [[ ${sourceCustomReleaseExitCode} -ne 0 ]]; then +# echo "Error while sourcing custom release arg file ${sourceCustomReleaseExitCode}. Exiting." +# exit 9 +# fi +# fi +#} +# +#PROJECT_DIR="$(pwd)" +#RELEASE_ARGS_FILE="${PROJECT_DIR}/release_args.sh" +# +#sourceCustomReleaseArgs "${RELEASE_ARGS_FILE}" + +source "$(pwd)/build/make/release_functions.sh" + +USERNAME="${1}" +PASSWORD="${2}" + +getActualCVEs "${USERNAME}" "${PASSWORD}" + +# TODO If length(local) - update a file with the new version number # - update_versions_stage_modified_files - stage a modified file to prepare the file for the up-coming commit -update_versions(){ +update_versions() { local NEW_RELEASE_VERSION="${1}" if [[ $(type -t update_versions_modify_files) == function ]]; then @@ -92,7 +92,7 @@ update_versions(){ # Update version in dogu.json if [ -f "dogu.json" ]; then echo "Updating version in dogu.json..." - jq ".Version = \"${NEW_RELEASE_VERSION}\"" dogu.json > dogu2.json && mv dogu2.json dogu.json + jq ".Version = \"${NEW_RELEASE_VERSION}\"" dogu.json >dogu2.json && mv dogu2.json dogu.json fi # Update version in Dockerfile @@ -110,7 +110,7 @@ update_versions(){ # Update version in package.json if [ -f "package.json" ]; then echo "Updating version in package.json..." - jq ".version = \"${NEW_RELEASE_VERSION}\"" package.json > package2.json && mv package2.json package.json + jq ".version = \"${NEW_RELEASE_VERSION}\"" package.json >package2.json && mv package2.json package.json fi # Update version in pom.xml @@ -133,7 +133,7 @@ update_versions(){ fi if [ -f "dogu.json" ]; then - git add dogu.json + git add dogu.json fi if [ -f "Dockerfile" ]; then @@ -155,7 +155,7 @@ update_versions(){ git commit -m "Bump version" } -update_changelog(){ +update_changelog() { local NEW_RELEASE_VERSION="${1}" # Changelog update @@ -186,8 +186,8 @@ update_changelog(){ git commit -m "Update changelog" } -show_diff(){ - if ! git diff --exit-code > /dev/null; then +show_diff() { + if ! git diff --exit-code >/dev/null; then echo "There are still uncommitted changes:" echo "" echo "# # # # # # # # # #" @@ -206,7 +206,7 @@ show_diff(){ echo "# # # # # # # # # #" } -finish_release_and_push(){ +finish_release_and_push() { local CURRENT_VERSION="${1}" local NEW_RELEASE_VERSION="${2}" @@ -218,3 +218,80 @@ finish_release_and_push(){ git checkout develop git branch -D release/v"${NEW_RELEASE_VERSION}" } + +LOCAL_TRIVY_CVE_LIST="" +REMOTE_TRIVY_CVE_LIST="" +#TRIVY_RESULT_PATH="/tmp/trivy" +TRIVY_RESULT_PATH="${PWD}/trivy" +TRIVY_RESULT_FILE="${TRIVY_RESULT_PATH}/results.json" + +getActualCVEs() { + mkdir -p "${TRIVY_RESULT_PATH}" + local USERNAME="${1}" + local PASSWORD="${2}" + dockerLogin "${USERNAME}" "${PASSWORD}" + pullRemoteImage + scanImage + parseTrivyJsonResult "local" + buildImage + scanImage + parseTrivyJsonResult "remote" + rm -rf "${TRIVY_RESULT_PATH}" + + echo "Remote CVES:\n" + echo "${REMOTE_TRIVY_CVE_LIST}" + echo "Local CVES:\n" + echo "${LOCAL_TRIVY_CVE_LIST}" +} + +dockerLogin() { + local USERNAME="${1}" + local PASSWORD="${2}" + docker login registry.cloudogu.com -u "${USERNAME}" -p "${PASSWORD}" +} + +pullRemoteImage() { + local IMAGE + local VERSION + IMAGE=$(jq -r .Image dogu.json) + VERSION=$(jq -r .Version dogu.json) + docker pull "${IMAGE}:${VERSION}" +} + +buildImage() { + IMAGE=$(jq -r .Image dogu.json) + VERSION=$(jq -r .Version dogu.json) + docker build . -t "${IMAGE}:${VERSION}" +} + +scanImage() { + local IMAGE + local VERSION + IMAGE=$(jq -r .Image dogu.json) + VERSION=$(jq -r .Version dogu.json) + # TODO save db in volume to reuse it. + docker run -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_RESULT_PATH}":/result aquasec/trivy -f json -o /result/results.json image "${IMAGE}:${VERSION}" +} + +parseTrivyJsonResult() { + local DESTINATION_RESULT="${1}" + + IFS=$'\n' + RESULTS_WITH_VULNS=$(cat "${TRIVY_RESULT_FILE}" | jq -c '.Results[] | select(.Vulnerabilities)') + echo "Extract CVE IDs" + CVE_LIST="" + for VULNS in $(echo "${RESULTS_WITH_VULNS}" | jq -c .Vulnerabilities); do + for VULN in $(echo "${VULNS}" | jq -c .[]); do + # TODO filter level of cves + ID="$(echo "${VULN}" | jq -rc .VulnerabilityID)" + CVE_LIST+="${ID} " + done + done + unset IFS + + if [[ "${DESTINATION_RESULT}" == "local" ]]; then + LOCAL_TRIVY_CVE_LIST="${CVE_LIST}" + elif [[ "${DESTINATION_RESULT}" == "remote" ]]; then + REMOTE_TRIVY_CVE_LIST="${CVE_LIST}" + fi +} From 495c5834639dc77dd12f3c672f69d68e25a221fa Mon Sep 17 00:00:00 2001 From: Niklas Date: Mon, 11 Sep 2023 11:31:32 +0200 Subject: [PATCH 02/17] #143 Cache the trivy db to save time --- build/make/release_functions.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build/make/release_functions.sh b/build/make/release_functions.sh index f75b620..903c932 100755 --- a/build/make/release_functions.sh +++ b/build/make/release_functions.sh @@ -221,12 +221,14 @@ finish_release_and_push() { LOCAL_TRIVY_CVE_LIST="" REMOTE_TRIVY_CVE_LIST="" -#TRIVY_RESULT_PATH="/tmp/trivy" -TRIVY_RESULT_PATH="${PWD}/trivy" -TRIVY_RESULT_FILE="${TRIVY_RESULT_PATH}/results.json" +#TRIVY_PATH="/tmp/trivy" +TRIVY_PATH="${PWD}/trivy" +TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" +TRIVY_CACHE_DIR="${TRIVY_PATH}/db" +TRIVY_DOCKER_CACHE_DIR=/tmp/db getActualCVEs() { - mkdir -p "${TRIVY_RESULT_PATH}" + mkdir -p "${TRIVY_PATH}" local USERNAME="${1}" local PASSWORD="${2}" dockerLogin "${USERNAME}" "${PASSWORD}" @@ -236,7 +238,7 @@ getActualCVEs() { buildImage scanImage parseTrivyJsonResult "remote" - rm -rf "${TRIVY_RESULT_PATH}" + #rm -rf "${TRIVY_PATH}" echo "Remote CVES:\n" echo "${REMOTE_TRIVY_CVE_LIST}" @@ -269,8 +271,7 @@ scanImage() { local VERSION IMAGE=$(jq -r .Image dogu.json) VERSION=$(jq -r .Version dogu.json) - # TODO save db in volume to reuse it. - docker run -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_RESULT_PATH}":/result aquasec/trivy -f json -o /result/results.json image "${IMAGE}:${VERSION}" + docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image "${IMAGE}:${VERSION}" } parseTrivyJsonResult() { @@ -278,7 +279,6 @@ parseTrivyJsonResult() { IFS=$'\n' RESULTS_WITH_VULNS=$(cat "${TRIVY_RESULT_FILE}" | jq -c '.Results[] | select(.Vulnerabilities)') - echo "Extract CVE IDs" CVE_LIST="" for VULNS in $(echo "${RESULTS_WITH_VULNS}" | jq -c .Vulnerabilities); do for VULN in $(echo "${VULNS}" | jq -c .[]); do From a984be279d41a84c78193b4864d60ec99270136c Mon Sep 17 00:00:00 2001 From: Niklas Date: Tue, 12 Sep 2023 08:19:53 +0200 Subject: [PATCH 03/17] #143 Add cve severity filter --- build/make/re-release.sh | 2 +- build/make/release_functions.sh | 27 ++++++++++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/build/make/re-release.sh b/build/make/re-release.sh index 7d7b9bf..a969354 100755 --- a/build/make/re-release.sh +++ b/build/make/re-release.sh @@ -35,7 +35,7 @@ source "$(pwd)/build/make/release_functions.sh" USERNAME="${1}" PASSWORD="${2}" -getActualCVEs "${USERNAME}" "${PASSWORD}" +getActualCVEs "${USERNAME}" "${PASSWORD}" "LOW" # TODO If length(local) Date: Tue, 12 Sep 2023 17:00:28 +0200 Subject: [PATCH 04/17] #143 Refactoring - determine if release is needed (wip) --- build/make/re-release.sh | 93 +++++++++++++++++--------------- build/make/release.sh | 8 +-- build/make/release_functions.sh | 95 +++++++++++++++++---------------- 3 files changed, 103 insertions(+), 93 deletions(-) diff --git a/build/make/re-release.sh b/build/make/re-release.sh index a969354..a5e8011 100755 --- a/build/make/re-release.sh +++ b/build/make/re-release.sh @@ -3,59 +3,64 @@ set -o errexit set -o nounset set -o pipefail -# Extension points in release.sh: -# -# A custom release argument file will be sourced if found. The custom release arg file may implement one or more bash -# functions which either release.sh or release_functions.sh define. If such a custom release function is found the -# release script must define the argument list which the custom release function will receive during the release. - -#sourceCustomReleaseArgs() { -# RELEASE_ARGS_FILE="${1}" -# -# if [[ -f "${RELEASE_ARGS_FILE}" ]]; then -# echo "Using custom release args file ${RELEASE_ARGS_FILE}" -# -# sourceCustomReleaseExitCode=0 -# # shellcheck disable=SC1090 -# source "${RELEASE_ARGS_FILE}" || sourceCustomReleaseExitCode=$? -# if [[ ${sourceCustomReleaseExitCode} -ne 0 ]]; then -# echo "Error while sourcing custom release arg file ${sourceCustomReleaseExitCode}. Exiting." -# exit 9 -# fi -# fi -#} -# -#PROJECT_DIR="$(pwd)" -#RELEASE_ARGS_FILE="${PROJECT_DIR}/release_args.sh" -# -#sourceCustomReleaseArgs "${RELEASE_ARGS_FILE}" - source "$(pwd)/build/make/release_functions.sh" USERNAME="${1}" PASSWORD="${2}" -getActualCVEs "${USERNAME}" "${PASSWORD}" "LOW" +#getRemoteAndLocalCVEList "${USERNAME}" "${PASSWORD}" + +### Testdata +#LOCAL_TRIVY_CVE_LIST_MEDIUM="CVE-1 CVE-2 CVE-2 CVE-3 CVE-4" +#REMOTE_TRIVY_CVE_LIST_MEDIUM="CVE-1 CVE-2 CVE-3 CVE-4" + +function intersectArrays { + local I=("$1") + local J=("$2") + local RESULT=() + + for i in ${I}; do + found=0 + for j in ${J}; do + [[ "${j}" == "${i}" ]] && { found=1; break; } + done + + [[ $found == 0 ]] && RESULT+=("$i") + done + + echo "${RESULT[@]}" +} + +CVE_IN_REMOTE_BUT_NOT_LOCAL=$(intersectArrays "${REMOTE_TRIVY_CVE_LIST_MEDIUM}" "${LOCAL_TRIVY_CVE_LIST_MEDIUM}") +CVE_IN_LOCAL_BUT_NOT_REMOTE=$(intersectArrays "${LOCAL_TRIVY_CVE_LIST_MEDIUM}" "${REMOTE_TRIVY_CVE_LIST_MEDIUM}") + +if [[ -n "${CVE_IN_LOCAL_BUT_NOT_REMOTE}" ]]; then + echo "Added new vulnerabilities:" + echo "${CVE_IN_LOCAL_BUT_NOT_REMOTE[@]}" + exit 2 +fi + +if [[ -z "${CVE_IN_REMOTE_BUT_NOT_LOCAL}" ]]; then + echo "Fixed no new vulnerabilities" + exit 3 +fi -# TODO If length(local) replace with ### Fixed\n-fix CVE-123,CVE-345... + # TODO If not Replace [Unreleased] with [Unreleased]\n###Fixed\n-fix CVE... + # Add new title line to changelog sed -i "s|## \[Unreleased\]|## \[Unreleased\]\n\n${NEW_CHANGELOG_TITLE}|g" CHANGELOG.md @@ -219,32 +223,44 @@ finish_release_and_push() { git branch -D release/v"${NEW_RELEASE_VERSION}" } -LOCAL_TRIVY_CVE_LIST="" -REMOTE_TRIVY_CVE_LIST="" +LOCAL_TRIVY_CVE_LIST_CRITICAL="" +LOCAL_TRIVY_CVE_LIST_HIGH="" +LOCAL_TRIVY_CVE_LIST_MEDIUM="" +LOCAL_TRIVY_CVE_LIST_LOW="" +REMOTE_TRIVY_CVE_LIST_CRITICAL="" +REMOTE_TRIVY_CVE_LIST_HIGH="" +REMOTE_TRIVY_CVE_LIST_MEDIUM="" +REMOTE_TRIVY_CVE_LIST_LOW="" #TRIVY_PATH="/tmp/trivy" TRIVY_PATH="${PWD}/trivy" TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" TRIVY_CACHE_DIR="${TRIVY_PATH}/db" TRIVY_DOCKER_CACHE_DIR=/tmp/db -getActualCVEs() { +getRemoteAndLocalCVEList() { mkdir -p "${TRIVY_PATH}" local USERNAME="${1}" local PASSWORD="${2}" - local SEVERITY="${3}" dockerLogin "${USERNAME}" "${PASSWORD}" - pullRemoteImage - scanImage - parseTrivyJsonResult "local" "${SEVERITY}" - buildImage - scanImage - parseTrivyJsonResult "remote" "${SEVERITY}" + getImageCVEListByType "local" + getImageCVEListByType "remote" #rm -rf "${TRIVY_PATH}" +} + +getImageCVEListByType() { + local TYPE="${1}" + mkdir -p "${TRIVY_PATH}" + if [[ "${TYPE}" == "local" ]]; then + buildLocalImage + elif [[ "${TYPE}" == "remote" ]]; then + pullRemoteImage + else + echo "Unknown type. Use local or remote" + exit 1 + fi - echo "Remote CVES:\n" - echo "${REMOTE_TRIVY_CVE_LIST}" - echo "Local CVES:\n" - echo "${LOCAL_TRIVY_CVE_LIST}" + scanImage + parseTrivyJsonResult "${TYPE}" } dockerLogin() { @@ -261,7 +277,7 @@ pullRemoteImage() { docker pull "${IMAGE}:${VERSION}" } -buildImage() { +buildLocalImage() { IMAGE=$(jq -r .Image dogu.json) VERSION=$(jq -r .Version dogu.json) docker build . -t "${IMAGE}:${VERSION}" @@ -272,43 +288,30 @@ scanImage() { local VERSION IMAGE=$(jq -r .Image dogu.json) VERSION=$(jq -r .Version dogu.json) - docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image "${IMAGE}:${VERSION}" + docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image "${IMAGE}:${VERSION}" } parseTrivyJsonResult() { local DESTINATION_RESULT="${1}" - local SEVERITY="${2}" - - local JQ_SEVERITY_FILTER="" - if [[ "${SEVERITY}" == "CRITICAL" ]]; then - JQ_SEVERITY_FILTER='select(.Severity == "CRITICAL")' - elif [[ "${SEVERITY}" == "HIGH" ]]; then - JQ_SEVERITY_FILTER='select(.Severity == "CRITICAL" or .Severity == "HIGH")' - elif [[ "${SEVERITY}" == "MEDIUM" ]]; then - JQ_SEVERITY_FILTER='select(.Severity == "CRITICAL" or .Severity == "HIGH" or .Severity == "MEDIUM")' - elif [[ "${SEVERITY}" == "LOW" ]]; then - JQ_SEVERITY_FILTER='select(.Severity == "CRITICAL" or .Severity == "HIGH" or .Severity == "MEDIUM" or .Severity == "LOW")' - else - echo "Unknown severity level: ${SEVERITY_FILTER}" - exit 1 - fi - IFS=$'\n' - local RESULTS_WITH_VULNS - RESULTS_WITH_VULNS=$(cat "${TRIVY_RESULT_FILE}" | jq -c '.Results[] | select(.Vulnerabilities)') - local CVE_LIST="" - for VULNS in $(echo "${RESULTS_WITH_VULNS}" | jq -c .Vulnerabilities); do - for VULN in $(echo "${VULNS}" | jq -c .[]); do - local ID - ID=$(echo "${VULN}" | jq -rc "${JQ_SEVERITY_FILTER} | .VulnerabilityID") - CVE_LIST+="${ID} " - done - done - unset IFS + local CRITICAL_CVE_LIST + CRITICAL_CVE_LIST=$(cat "${TRIVY_RESULT_FILE}" | jq -rc '[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == "CRITICAL") | .VulnerabilityID] | join(" ")') + local HIGH_CVE_LIST + HIGH_CVE_LIST=$(cat "${TRIVY_RESULT_FILE}" | jq -rc '[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == "HIGH") | .VulnerabilityID] | join(" ")') + local MEDIUM_CVE_LIST + MEDIUM_CVE_LIST=$(cat "${TRIVY_RESULT_FILE}" | jq -rc '[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == "MEDIUM") | .VulnerabilityID] | join(" ")') + local LOW_CVE_LIST + LOW_CVE_LIST=$(cat "${TRIVY_RESULT_FILE}" | jq -rc '[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == "LOW") | .VulnerabilityID] | join(" ")') if [[ "${DESTINATION_RESULT}" == "local" ]]; then - LOCAL_TRIVY_CVE_LIST="${CVE_LIST}" - elif [[ "${DESTINATION_RESULT}" == "remote" ]]; then - REMOTE_TRIVY_CVE_LIST="${CVE_LIST}" +# LOCAL_TRIVY_CVE_LIST_HIGH="${HIGH_CVE_LIST}" +# LOCAL_TRIVY_CVE_LIST_CRITICAL="${CRITICAL_CVE_LIST}" + LOCAL_TRIVY_CVE_LIST_MEDIUM="${MEDIUM_CVE_LIST}" +# LOCAL_TRIVY_CVE_LIST_LOW="${LOW_CVE_LIST}" + elif [[ "${DESTINATION_RESULT}" == "remote" ]]; then +# REMOTE_TRIVY_CVE_LIST_HIGH="${HIGH_CVE_LIST}" +# REMOTE_TRIVY_CVE_LIST_CRITICAL="${CVE_LIST_CRITICAL}" + REMOTE_TRIVY_CVE_LIST_MEDIUM="${MEDIUM_CVE_LIST}" +# REMOTE_TRIVY_CVE_LIST_LOW="${LOW_CVE_LIST}" fi } From 0b41b1362c5500d84377793b4130843816dcf604 Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 13 Sep 2023 15:18:58 +0200 Subject: [PATCH 05/17] #143 Refactoring - Add changelog entry with fixed cves - restructure code - start release process - read credentials optionally if not set --- build/make/release.mk | 6 +- build/make/release_cve.sh | 122 +++++++++++++++++++++++++++++++ build/make/release_functions.sh | 123 ++++++++------------------------ 3 files changed, 153 insertions(+), 98 deletions(-) create mode 100755 build/make/release_cve.sh diff --git a/build/make/release.mk b/build/make/release.mk index 5d1c1dd..14468ce 100644 --- a/build/make/release.mk +++ b/build/make/release.mk @@ -10,6 +10,6 @@ dogu-release: ## Start a dogu release go-release: ## Start a go tool release build/make/release.sh go-tool -.PHONY: dogu-re-release -dogu-re-release: ## Start a dogu re release of a new build fix CVEs - @build/make/re-release.sh "${REGISTRY_USERNAME}" "${REGISTRY_PASSWORD}" \ No newline at end of file +.PHONY: dogu-cve-release +dogu-cve-release: ## Start a dogu release of a new build if the local build fixes critical CVEs + @build/make/release_cve.sh "${REGISTRY_USERNAME}" "${REGISTRY_PASSWORD}" \ No newline at end of file diff --git a/build/make/release_cve.sh b/build/make/release_cve.sh new file mode 100755 index 0000000..5f76fe7 --- /dev/null +++ b/build/make/release_cve.sh @@ -0,0 +1,122 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +function diffArrays { + local I=("$1") + local J=("$2") + local RESULT=() + + for i in ${I}; do + local FOUND=0 + for j in ${J}; do + [[ "${j}" == "${i}" ]] && { + FOUND=1 + break + } + done + + [[ $FOUND == 0 ]] && RESULT+=("$i") + done + + echo "${RESULT[@]}" +} + +dockerLogin() { + local USERNAME="${1}" + local PASSWORD="${2}" + docker login "${REGISTRY_URL}" -u "${USERNAME}" -p "${PASSWORD}" +} + +dockerLogout() { + docker logout "${REGISTRY_URL}" +} + +pullRemoteImage() { + local IMAGE + local VERSION + IMAGE=$(jq -r .Image dogu.json) + VERSION=$(jq -r .Version dogu.json) + docker pull "${IMAGE}:${VERSION}" +} + +buildLocalImage() { + local IMAGE + local VERSION + IMAGE=$(jq -r .Image dogu.json) + VERSION=$(jq -r .Version dogu.json) + docker build . -t "${IMAGE}:${VERSION}" +} + +scanImage() { + local IMAGE + local VERSION + IMAGE=$(jq -r .Image dogu.json) + VERSION=$(jq -r .Version dogu.json) + docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image "${IMAGE}:${VERSION}" +} + +parseTrivyJsonResult() { + local SEVERITY="${1}" + local TRIVY_RESULT_FILE="${2}" + local CVE_RESULT="" + CVE_RESULT=$(jq -rc "[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"${SEVERITY}\") | .VulnerabilityID] | join(\" \")" "${TRIVY_RESULT_FILE}" ) + echo "${CVE_RESULT}" +} + +REGISTRY_URL="registry.cloudogu.com" +LOCAL_TRIVY_CVE_LIST_CRITICAL="" +REMOTE_TRIVY_CVE_LIST_CRITICAL="" + +CVE_SEVERITY="CRITICAL" + +TRIVY_PATH="/tmp/trivy-dogu-cve-release" +TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" +TRIVY_CACHE_DIR="${TRIVY_PATH}/db" +TRIVY_DOCKER_CACHE_DIR=/tmp/db + +USERNAME="${1}" +PASSWORD="${2}" + +if [ -z "${USERNAME}" ]; then + echo "username is unset" + while [[ -z ${USERNAME} ]]; do + read -r -p "type username for ${REGISTRY_URL}: " USERNAME + done +fi +if [ -z "${PASSWORD}" ]; then + echo "password is unset" + while [[ -z ${PASSWORD} ]]; do + read -r -s -p "type password for ${REGISTRY_URL}: " PASSWORD + done +fi + +dockerLogin "${USERNAME}" "${PASSWORD}" + +mkdir -p "${TRIVY_PATH}" # Cache will not be removed after release. rm requires root because the trivy container only runs with root. +pullRemoteImage +scanImage +parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}" + +buildLocalImage +scanImage +parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}" + +dockerLogout + +CVE_IN_REMOTE_BUT_NOT_LOCAL=$(diffArrays "${REMOTE_TRIVY_CVE_LIST_CRITICAL}" "${LOCAL_TRIVY_CVE_LIST_CRITICAL}") +CVE_IN_LOCAL_BUT_NOT_REMOTE=$(diffArrays "${LOCAL_TRIVY_CVE_LIST_CRITICAL}" "${REMOTE_TRIVY_CVE_LIST_CRITICAL}") + +if [[ -n "${CVE_IN_LOCAL_BUT_NOT_REMOTE}" ]]; then + echo "Abort release. Added new vulnerabilities:" + echo "${CVE_IN_LOCAL_BUT_NOT_REMOTE[@]}" + exit 2 +fi + +if [[ -z "${CVE_IN_REMOTE_BUT_NOT_LOCAL}" ]]; then + echo "Abort release. Fixed no new vulnerabilities" + exit 3 +fi + +build/make/release.sh "dogu-cve-release" "${CVE_IN_REMOTE_BUT_NOT_LOCAL}" diff --git a/build/make/release_functions.sh b/build/make/release_functions.sh index c966a73..8856738 100755 --- a/build/make/release_functions.sh +++ b/build/make/release_functions.sh @@ -170,8 +170,10 @@ update_changelog() { wait_for_ok "Please insert a \"## [Unreleased]\" line into CHANGELOG.md now." done - # TODO If ### Fixed exist -> replace with ### Fixed\n-fix CVE-123,CVE-345... - # TODO If not Replace [Unreleased] with [Unreleased]\n###Fixed\n-fix CVE... + if [[ -n "${FIXED_CVE_LIST}" ]]; then + # Add fixed CVEs in unreleased section + addFixedCVEListFromReRelease "${FIXED_CVE_LIST}" + fi # Add new title line to changelog sed -i "s|## \[Unreleased\]|## \[Unreleased\]\n\n${NEW_CHANGELOG_TITLE}|g" CHANGELOG.md @@ -190,6 +192,30 @@ update_changelog() { git commit -m "Update changelog" } +addFixedCVEListFromReRelease() { + local FIXED_CVE_LIST="${1}" + + local CVE_SED_SEARCH="" + local CVE_SED_REPLACE="" + local FIXED_EXISTS_IN_UNRELEASED + FIXED_EXISTS_IN_UNRELEASED=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^### Fixed$" || true) + if [[ -n "${FIXED_EXISTS_IN_UNRELEASED}" ]]; then + CVE_SED_SEARCH="^\#\#\# Fixed$" + CVE_SED_REPLACE="\#\#\# Fixed\n- Fixed ${FIXED_CVE_LIST}" + else + CVE_SED_SEARCH="^\#\# \[Unreleased\]$" + CVE_SED_REPLACE="\#\# \[Unreleased\]\n\#\#\# Fixed\n- Fixed ${FIXED_CVE_LIST}" + + local ANY_EXISTS_UNRELEASED + ANY_EXISTS_UNRELEASED=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^\#\#\# Added$" -e "^\#\#\# Fixed$" -e "^\#\#\# Changed$" || true) + if [[ -n ${ANY_EXISTS_UNRELEASED} ]]; then + CVE_SED_REPLACE+="\n" + fi + fi + + sed -i "0,/${CVE_SED_SEARCH}/s//${CVE_SED_REPLACE}/" CHANGELOG.md +} + show_diff() { if ! git diff --exit-code >/dev/null; then echo "There are still uncommitted changes:" @@ -222,96 +248,3 @@ finish_release_and_push() { git checkout develop git branch -D release/v"${NEW_RELEASE_VERSION}" } - -LOCAL_TRIVY_CVE_LIST_CRITICAL="" -LOCAL_TRIVY_CVE_LIST_HIGH="" -LOCAL_TRIVY_CVE_LIST_MEDIUM="" -LOCAL_TRIVY_CVE_LIST_LOW="" -REMOTE_TRIVY_CVE_LIST_CRITICAL="" -REMOTE_TRIVY_CVE_LIST_HIGH="" -REMOTE_TRIVY_CVE_LIST_MEDIUM="" -REMOTE_TRIVY_CVE_LIST_LOW="" -#TRIVY_PATH="/tmp/trivy" -TRIVY_PATH="${PWD}/trivy" -TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" -TRIVY_CACHE_DIR="${TRIVY_PATH}/db" -TRIVY_DOCKER_CACHE_DIR=/tmp/db - -getRemoteAndLocalCVEList() { - mkdir -p "${TRIVY_PATH}" - local USERNAME="${1}" - local PASSWORD="${2}" - dockerLogin "${USERNAME}" "${PASSWORD}" - getImageCVEListByType "local" - getImageCVEListByType "remote" - #rm -rf "${TRIVY_PATH}" -} - -getImageCVEListByType() { - local TYPE="${1}" - mkdir -p "${TRIVY_PATH}" - if [[ "${TYPE}" == "local" ]]; then - buildLocalImage - elif [[ "${TYPE}" == "remote" ]]; then - pullRemoteImage - else - echo "Unknown type. Use local or remote" - exit 1 - fi - - scanImage - parseTrivyJsonResult "${TYPE}" -} - -dockerLogin() { - local USERNAME="${1}" - local PASSWORD="${2}" - docker login registry.cloudogu.com -u "${USERNAME}" -p "${PASSWORD}" -} - -pullRemoteImage() { - local IMAGE - local VERSION - IMAGE=$(jq -r .Image dogu.json) - VERSION=$(jq -r .Version dogu.json) - docker pull "${IMAGE}:${VERSION}" -} - -buildLocalImage() { - IMAGE=$(jq -r .Image dogu.json) - VERSION=$(jq -r .Version dogu.json) - docker build . -t "${IMAGE}:${VERSION}" -} - -scanImage() { - local IMAGE - local VERSION - IMAGE=$(jq -r .Image dogu.json) - VERSION=$(jq -r .Version dogu.json) - docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image "${IMAGE}:${VERSION}" -} - -parseTrivyJsonResult() { - local DESTINATION_RESULT="${1}" - - local CRITICAL_CVE_LIST - CRITICAL_CVE_LIST=$(cat "${TRIVY_RESULT_FILE}" | jq -rc '[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == "CRITICAL") | .VulnerabilityID] | join(" ")') - local HIGH_CVE_LIST - HIGH_CVE_LIST=$(cat "${TRIVY_RESULT_FILE}" | jq -rc '[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == "HIGH") | .VulnerabilityID] | join(" ")') - local MEDIUM_CVE_LIST - MEDIUM_CVE_LIST=$(cat "${TRIVY_RESULT_FILE}" | jq -rc '[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == "MEDIUM") | .VulnerabilityID] | join(" ")') - local LOW_CVE_LIST - LOW_CVE_LIST=$(cat "${TRIVY_RESULT_FILE}" | jq -rc '[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == "LOW") | .VulnerabilityID] | join(" ")') - - if [[ "${DESTINATION_RESULT}" == "local" ]]; then -# LOCAL_TRIVY_CVE_LIST_HIGH="${HIGH_CVE_LIST}" -# LOCAL_TRIVY_CVE_LIST_CRITICAL="${CRITICAL_CVE_LIST}" - LOCAL_TRIVY_CVE_LIST_MEDIUM="${MEDIUM_CVE_LIST}" -# LOCAL_TRIVY_CVE_LIST_LOW="${LOW_CVE_LIST}" - elif [[ "${DESTINATION_RESULT}" == "remote" ]]; then -# REMOTE_TRIVY_CVE_LIST_HIGH="${HIGH_CVE_LIST}" -# REMOTE_TRIVY_CVE_LIST_CRITICAL="${CVE_LIST_CRITICAL}" - REMOTE_TRIVY_CVE_LIST_MEDIUM="${MEDIUM_CVE_LIST}" -# REMOTE_TRIVY_CVE_LIST_LOW="${LOW_CVE_LIST}" - fi -} From bc369438da33d58abcd87438f4f8aa23ef1b4fce Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 13 Sep 2023 15:38:19 +0200 Subject: [PATCH 06/17] #143 Refactoring - Add changelog entry with fixed cves - restructure code - start release process - read credentials optionally if not set --- README.md | 2 + build/make/re-release.sh | 66 --------------------------------- build/make/release.sh | 7 ++-- build/make/release_cve.sh | 17 +++++---- build/make/release_functions.sh | 13 +++++-- 5 files changed, 26 insertions(+), 79 deletions(-) delete mode 100755 build/make/re-release.sh diff --git a/README.md b/README.md index 5127c68..f950a2b 100644 --- a/README.md +++ b/README.md @@ -284,5 +284,7 @@ This module enables you to use bower via the `bower-install` target. ### release.mk This module holds the `dogu-release` or other binary release related targets for starting automated production releases. +Additionally, to the regular `dogu-release` the module contains a `dogu-cve-release`. This target checks if a simple +build of a dogu eliminates critical CVEs. If yes a release process will be triggered. Only include this module in dogu or Golang repositories that support a dedicated release flow! diff --git a/build/make/re-release.sh b/build/make/re-release.sh deleted file mode 100755 index a5e8011..0000000 --- a/build/make/re-release.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - -source "$(pwd)/build/make/release_functions.sh" - -USERNAME="${1}" -PASSWORD="${2}" - -#getRemoteAndLocalCVEList "${USERNAME}" "${PASSWORD}" - -### Testdata -#LOCAL_TRIVY_CVE_LIST_MEDIUM="CVE-1 CVE-2 CVE-2 CVE-3 CVE-4" -#REMOTE_TRIVY_CVE_LIST_MEDIUM="CVE-1 CVE-2 CVE-3 CVE-4" - -function intersectArrays { - local I=("$1") - local J=("$2") - local RESULT=() - - for i in ${I}; do - found=0 - for j in ${J}; do - [[ "${j}" == "${i}" ]] && { found=1; break; } - done - - [[ $found == 0 ]] && RESULT+=("$i") - done - - echo "${RESULT[@]}" -} - -CVE_IN_REMOTE_BUT_NOT_LOCAL=$(intersectArrays "${REMOTE_TRIVY_CVE_LIST_MEDIUM}" "${LOCAL_TRIVY_CVE_LIST_MEDIUM}") -CVE_IN_LOCAL_BUT_NOT_REMOTE=$(intersectArrays "${LOCAL_TRIVY_CVE_LIST_MEDIUM}" "${REMOTE_TRIVY_CVE_LIST_MEDIUM}") - -if [[ -n "${CVE_IN_LOCAL_BUT_NOT_REMOTE}" ]]; then - echo "Added new vulnerabilities:" - echo "${CVE_IN_LOCAL_BUT_NOT_REMOTE[@]}" - exit 2 -fi - -if [[ -z "${CVE_IN_REMOTE_BUT_NOT_LOCAL}" ]]; then - echo "Fixed no new vulnerabilities" - exit 3 -fi - - -echo "=====Starting Release process=====" - -if [ "${TYPE}" == "dogu" ];then - CURRENT_TOOL_VERSION=$(get_current_version_by_dogu_json) -else - CURRENT_TOOL_VERSION=$(get_current_version_by_makefile) -fi - -NEW_RELEASE_VERSION="$(read_new_version)" - -validate_new_version "${NEW_RELEASE_VERSION}" -start_git_flow_release "${NEW_RELEASE_VERSION}" -update_versions "${NEW_RELEASE_VERSION}" -update_changelog "${NEW_RELEASE_VERSION}" -show_diff -#finish_release_and_push "${CURRENT_TOOL_VERSION}" "${NEW_RELEASE_VERSION}" - -echo "=====Finished Release process=====" diff --git a/build/make/release.sh b/build/make/release.sh index b1b2357..c0eef3f 100755 --- a/build/make/release.sh +++ b/build/make/release.sh @@ -15,7 +15,7 @@ sourceCustomReleaseArgs() { if [[ -f "${RELEASE_ARGS_FILE}" ]]; then echo "Using custom release args file ${RELEASE_ARGS_FILE}" - sourceCustomReleaseExitCode=0 + local sourceCustomReleaseExitCode=0 # shellcheck disable=SC1090 source "${RELEASE_ARGS_FILE}" || sourceCustomReleaseExitCode=$? if [[ ${sourceCustomReleaseExitCode} -ne 0 ]]; then @@ -30,10 +30,11 @@ RELEASE_ARGS_FILE="${PROJECT_DIR}/release_args.sh" sourceCustomReleaseArgs "${RELEASE_ARGS_FILE}" +# shellcheck disable=SC1090 source "$(pwd)/build/make/release_functions.sh" TYPE="${1}" -FIXED_CVE_LIST="${2}" +FIXED_CVE_LIST="${2:-""}" echo "=====Starting Release process=====" @@ -51,6 +52,6 @@ update_versions "${NEW_RELEASE_VERSION}" update_changelog "${NEW_RELEASE_VERSION}" "${FIXED_CVE_LIST}" show_diff -#finish_release_and_push "${CURRENT_TOOL_VERSION}" "${NEW_RELEASE_VERSION}" +finish_release_and_push "${CURRENT_TOOL_VERSION}" "${NEW_RELEASE_VERSION}" echo "=====Finished Release process=====" diff --git a/build/make/release_cve.sh b/build/make/release_cve.sh index 5f76fe7..1a65245 100755 --- a/build/make/release_cve.sh +++ b/build/make/release_cve.sh @@ -8,8 +8,11 @@ function diffArrays { local J=("$2") local RESULT=() + local i + # shellcheck disable=SC2128 for i in ${I}; do local FOUND=0 + local j for j in ${J}; do [[ "${j}" == "${i}" ]] && { FOUND=1 @@ -66,10 +69,10 @@ parseTrivyJsonResult() { } REGISTRY_URL="registry.cloudogu.com" -LOCAL_TRIVY_CVE_LIST_CRITICAL="" -REMOTE_TRIVY_CVE_LIST_CRITICAL="" +LOCAL_TRIVY_CVE_LIST="" +REMOTE_TRIVY_CVE_LIST="" -CVE_SEVERITY="CRITICAL" +CVE_SEVERITY="MEDIUM" TRIVY_PATH="/tmp/trivy-dogu-cve-release" TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" @@ -97,16 +100,16 @@ dockerLogin "${USERNAME}" "${PASSWORD}" mkdir -p "${TRIVY_PATH}" # Cache will not be removed after release. rm requires root because the trivy container only runs with root. pullRemoteImage scanImage -parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}" +REMOTE_TRIVY_CVE_LIST=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") buildLocalImage scanImage -parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}" +LOCAL_TRIVY_CVE_LIST=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") dockerLogout -CVE_IN_REMOTE_BUT_NOT_LOCAL=$(diffArrays "${REMOTE_TRIVY_CVE_LIST_CRITICAL}" "${LOCAL_TRIVY_CVE_LIST_CRITICAL}") -CVE_IN_LOCAL_BUT_NOT_REMOTE=$(diffArrays "${LOCAL_TRIVY_CVE_LIST_CRITICAL}" "${REMOTE_TRIVY_CVE_LIST_CRITICAL}") +CVE_IN_REMOTE_BUT_NOT_LOCAL=$(diffArrays "${REMOTE_TRIVY_CVE_LIST}" "${LOCAL_TRIVY_CVE_LIST}") +CVE_IN_LOCAL_BUT_NOT_REMOTE=$(diffArrays "${LOCAL_TRIVY_CVE_LIST}" "${REMOTE_TRIVY_CVE_LIST}") if [[ -n "${CVE_IN_LOCAL_BUT_NOT_REMOTE}" ]]; then echo "Abort release. Added new vulnerabilities:" diff --git a/build/make/release_functions.sh b/build/make/release_functions.sh index 8856738..43501c0 100755 --- a/build/make/release_functions.sh +++ b/build/make/release_functions.sh @@ -5,7 +5,7 @@ set -o pipefail wait_for_ok() { printf "\n" - OK=false + local OK=false while [[ ${OK} != "ok" ]]; do read -r -p "${1} (type 'ok'): " OK done @@ -41,6 +41,7 @@ validate_new_version() { if [[ ${NEW_RELEASE_VERSION} = v* ]]; then echo "WARNING: The new release version (v${NEW_RELEASE_VERSION}) starts with 'vv'." echo "You must not enter the v when defining the new version." + local ANSWER ANSWER=$(ask_yes_or_no "Should the first v be removed?") if [ "${ANSWER}" == "y" ]; then NEW_RELEASE_VERSION="${NEW_RELEASE_VERSION:1}" @@ -54,6 +55,7 @@ start_git_flow_release() { # Do gitflow git flow init --defaults --force + local mainBranchExists mainBranchExists="$(git show-ref refs/remotes/origin/main || echo "")" if [ -n "$mainBranchExists" ]; then echo 'Using "main" branch for production releases' @@ -81,7 +83,7 @@ update_versions() { local NEW_RELEASE_VERSION="${1}" if [[ $(type -t update_versions_modify_files) == function ]]; then - preSkriptExitCode=0 + local preSkriptExitCode=0 update_versions_modify_files "${NEW_RELEASE_VERSION}" || preSkriptExitCode=$? if [[ ${preSkriptExitCode} -ne 0 ]]; then echo "ERROR: custom update_versions_modify_files() exited with exit code ${preSkriptExitCode}" @@ -160,8 +162,9 @@ update_changelog() { local FIXED_CVE_LIST="${2}" # Changelog update + local CURRENT_DATE CURRENT_DATE=$(date --rfc-3339=date) - NEW_CHANGELOG_TITLE="## [v${NEW_RELEASE_VERSION}] - ${CURRENT_DATE}" + local NEW_CHANGELOG_TITLE="## [v${NEW_RELEASE_VERSION}] - ${CURRENT_DATE}" # Check if "Unreleased" tag exists while ! grep --silent "## \[Unreleased\]" CHANGELOG.md; do echo "" @@ -192,6 +195,8 @@ update_changelog() { git commit -m "Update changelog" } +# addFixedCVEListFromReRelease is used in dogu cve releases. The method adds the fixed CVEs under the ### Fixed header +# in the unreleased section. addFixedCVEListFromReRelease() { local FIXED_CVE_LIST="${1}" @@ -200,9 +205,11 @@ addFixedCVEListFromReRelease() { local FIXED_EXISTS_IN_UNRELEASED FIXED_EXISTS_IN_UNRELEASED=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^### Fixed$" || true) if [[ -n "${FIXED_EXISTS_IN_UNRELEASED}" ]]; then + # extend fixed header with CVEs. CVE_SED_SEARCH="^\#\#\# Fixed$" CVE_SED_REPLACE="\#\#\# Fixed\n- Fixed ${FIXED_CVE_LIST}" else + # extend unreleased header with fixed header and CVEs. CVE_SED_SEARCH="^\#\# \[Unreleased\]$" CVE_SED_REPLACE="\#\# \[Unreleased\]\n\#\#\# Fixed\n- Fixed ${FIXED_CVE_LIST}" From 071c4419e759e4f736e718b719ba0a7b8db56877 Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 13 Sep 2023 16:16:24 +0200 Subject: [PATCH 07/17] #143 Add changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b581ad8..45531c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- [#143] Add release target for dogus if a simple rebuild fixes critical CVEs. ## [v8.0.0](https://github.com/cloudogu/makefiles/releases/tag/v8.0.0) 2023-09-12 ### Added From 11519264b2e7662f9fca38fe84a68d52996a49a8 Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 13 Sep 2023 17:08:55 +0200 Subject: [PATCH 08/17] #143 Change severity to critical --- build/make/release_cve.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/make/release_cve.sh b/build/make/release_cve.sh index 1a65245..d5abf7a 100755 --- a/build/make/release_cve.sh +++ b/build/make/release_cve.sh @@ -72,7 +72,7 @@ REGISTRY_URL="registry.cloudogu.com" LOCAL_TRIVY_CVE_LIST="" REMOTE_TRIVY_CVE_LIST="" -CVE_SEVERITY="MEDIUM" +CVE_SEVERITY="CRITICAL" TRIVY_PATH="/tmp/trivy-dogu-cve-release" TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" From bff7fc506819f003ba230c8a8c690d3c2c6e494a Mon Sep 17 00:00:00 2001 From: Niklas Date: Thu, 14 Sep 2023 16:33:12 +0200 Subject: [PATCH 09/17] #143 Refactoring Use bash source guard and force the execution of the release.sh with bash to add bats tests. --- .gitignore | 2 + CHANGELOG.md | 2 +- Makefile | 1 + README.md | 2 +- batsTests/release_cve.bats | 28 +++++ build/make/release.mk | 2 +- build/make/release_cve.sh | 195 ++++++++++++++++++-------------- build/make/release_functions.sh | 35 +++--- 8 files changed, 158 insertions(+), 109 deletions(-) create mode 100644 batsTests/release_cve.bats diff --git a/.gitignore b/.gitignore index 485dee6..6dcf23c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ .idea +target +.bin diff --git a/CHANGELOG.md b/CHANGELOG.md index 45531c5..68c9e7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- [#143] Add release target for dogus if a simple rebuild fixes critical CVEs. +- [#143] Add release target `dogu-cve-release` for dogus if a simple rebuild fixes critical CVEs. ## [v8.0.0](https://github.com/cloudogu/makefiles/releases/tag/v8.0.0) 2023-09-12 ### Added diff --git a/Makefile b/Makefile index 65fc87c..795446e 100644 --- a/Makefile +++ b/Makefile @@ -57,4 +57,5 @@ include build/make/release.mk include build/make/k8s-dogu.mk # or k8s-controller.mk; only include this in k8s-controller repositories include build/make/k8s-controller.mk +include build/make/bats.mk diff --git a/README.md b/README.md index f950a2b..04aec02 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,6 @@ This module enables you to use bower via the `bower-install` target. This module holds the `dogu-release` or other binary release related targets for starting automated production releases. Additionally, to the regular `dogu-release` the module contains a `dogu-cve-release`. This target checks if a simple -build of a dogu eliminates critical CVEs. If yes a release process will be triggered. +build of a dogu eliminates critical CVEs. If this is the case, a release process will be triggered. Only include this module in dogu or Golang repositories that support a dedicated release flow! diff --git a/batsTests/release_cve.bats b/batsTests/release_cve.bats new file mode 100644 index 0000000..8fba18a --- /dev/null +++ b/batsTests/release_cve.bats @@ -0,0 +1,28 @@ +#! /bin/bash +# Bind an unbound BATS variables that fail all tests when combined with 'set -o nounset' +export BATS_TEST_START_TIME="0" +export BATSLIB_FILE_PATH_REM="" +export BATSLIB_FILE_PATH_ADD="" + +load '/workspace/target/bats_libs/bats-support/load.bash' +load '/workspace/target/bats_libs/bats-assert/load.bash' +load '/workspace/target/bats_libs/bats-mock/load.bash' +load '/workspace/target/bats_libs/bats-file/load.bash' + +setup() { + export WORKDIR=/workspace + export MAKE_DIR="${WORKDIR}"/build/make + export PATH="${PATH}:${BATS_TMPDIR}" +} + +teardown() { + unset MAKE_DIR + unset WORKDIR +} + +@test "source script with bash should return exit code 0" { + run source "${MAKE_DIR}/release_cve.sh" + + assert_success +} + diff --git a/build/make/release.mk b/build/make/release.mk index 14468ce..99da81c 100644 --- a/build/make/release.mk +++ b/build/make/release.mk @@ -12,4 +12,4 @@ go-release: ## Start a go tool release .PHONY: dogu-cve-release dogu-cve-release: ## Start a dogu release of a new build if the local build fixes critical CVEs - @build/make/release_cve.sh "${REGISTRY_USERNAME}" "${REGISTRY_PASSWORD}" \ No newline at end of file + @bash -c "build/make/release_cve.sh ${REGISTRY_USERNAME} ${REGISTRY_PASSWORD}" \ No newline at end of file diff --git a/build/make/release_cve.sh b/build/make/release_cve.sh index d5abf7a..8a248e6 100755 --- a/build/make/release_cve.sh +++ b/build/make/release_cve.sh @@ -1,125 +1,144 @@ #!/bin/bash set -o errexit -set -o nounset set -o pipefail +set -o nounset + +function readCredentialsIfUnset() { + if [ -z "${USERNAME}" ]; then + echo "username is unset" + while [[ -z ${USERNAME} ]]; do + read -r -p "type username for ${REGISTRY_URL}: " USERNAME + done + fi + if [ -z "${PASSWORD}" ]; then + echo "password is unset" + while [[ -z ${PASSWORD} ]]; do + read -r -s -p "type password for ${REGISTRY_URL}: " PASSWORD + done + fi +} -function diffArrays { - local I=("$1") - local J=("$2") - local RESULT=() +function diffArrays() { + local cveListX=("$1") + local cveListY=("$2") + local result=() - local i + local cveX + # Disable the following shellcheck because the arrays are sufficiently whitespace delimited because of the jq parsing result. # shellcheck disable=SC2128 - for i in ${I}; do - local FOUND=0 - local j - for j in ${J}; do - [[ "${j}" == "${i}" ]] && { - FOUND=1 + for cveX in "${cveListX[@]}"; do + local found=0 + local cveY + echo "$cveX" + for cveY in ${cveListY}; do + [[ "${cveY}" == "${cveX}" ]] && { + found=1 break } done - [[ $FOUND == 0 ]] && RESULT+=("$i") + [[ "${found}" == 0 ]] && result+=("${cveX}") done - echo "${RESULT[@]}" + echo "${result[@]}" } -dockerLogin() { - local USERNAME="${1}" - local PASSWORD="${2}" +function dockerLogin() { docker login "${REGISTRY_URL}" -u "${USERNAME}" -p "${PASSWORD}" } -dockerLogout() { +function dockerLogout() { docker logout "${REGISTRY_URL}" } -pullRemoteImage() { - local IMAGE - local VERSION - IMAGE=$(jq -r .Image dogu.json) - VERSION=$(jq -r .Version dogu.json) - docker pull "${IMAGE}:${VERSION}" +function nameFromDogu() { + jsonPropertyFromDogu ".Name" +} + +function imageFromDogu() { + jsonPropertyFromDogu ".Image" } -buildLocalImage() { - local IMAGE - local VERSION - IMAGE=$(jq -r .Image dogu.json) - VERSION=$(jq -r .Version dogu.json) - docker build . -t "${IMAGE}:${VERSION}" +function versionsFromDogu() { + jsonPropertyFromDogu ".Version" } -scanImage() { - local IMAGE - local VERSION - IMAGE=$(jq -r .Image dogu.json) - VERSION=$(jq -r .Version dogu.json) - docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image "${IMAGE}:${VERSION}" +function jsonPropertyFromDogu() { + local property="${1}" + jq -r "${property}" "${DOGU_JSON_FILE}" } -parseTrivyJsonResult() { - local SEVERITY="${1}" - local TRIVY_RESULT_FILE="${2}" - local CVE_RESULT="" - CVE_RESULT=$(jq -rc "[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"${SEVERITY}\") | .VulnerabilityID] | join(\" \")" "${TRIVY_RESULT_FILE}" ) - echo "${CVE_RESULT}" +function pullRemoteImage() { + docker pull "$(imageFromDogu):$(versionFromDogu)" +} + +function buildLocalImage() { + docker build . -t "$(imageFromDogu):$(versionFromDogu)" +} + +function scanImage() { + docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image "${TRIVY_IMAGE_SCAN_FLAGS}" "$(imageFromDogu):$(versionFromDogu)" +} + +function parseTrivyJsonResult() { + local severity="${1}" + local trivy_result_file="${2}" + local cve_result="" + cve_result=$(jq -rc "[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"${severity}\") | .VulnerabilityID] | join(\" \")" "${trivy_result_file}") + echo "${cve_result}" } REGISTRY_URL="registry.cloudogu.com" -LOCAL_TRIVY_CVE_LIST="" -REMOTE_TRIVY_CVE_LIST="" +DOGU_JSON_FILE="dogu.json" CVE_SEVERITY="CRITICAL" -TRIVY_PATH="/tmp/trivy-dogu-cve-release" +TRIVY_PATH="/tmp/trivy-dogu-cve-release-$(nameFromDogu)" TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" TRIVY_CACHE_DIR="${TRIVY_PATH}/db" TRIVY_DOCKER_CACHE_DIR=/tmp/db +TRIVY_IMAGE_SCAN_FLAGS= + +USERNAME="" +PASSWORD="" + +function run() { + readCredentialsIfUnset + dockerLogin + + mkdir -p "${TRIVY_PATH}" # Cache will not be removed after release. rm requires root because the trivy container only runs with root. + pullRemoteImage + scanImage + remote_trivy_cve_list=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") + + buildLocalImage + scanImage + local_trivy_cve_list=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") + + dockerLogout + + local cve_in_local_but_not_in_remote + cve_in_local_but_not_in_remote=$(diffArrays "${local_trivy_cve_list}" "${remote_trivy_cve_list}") + if [[ -n "${cve_in_local_but_not_in_remote}" ]]; then + echo "Abort release. Added new vulnerabilities:" + echo "${cve_in_local_but_not_in_remote[@]}" + exit 2 + fi + + local cve_in_remote_but_not_in_local + cve_in_remote_but_not_in_local=$(diffArrays "${remote_trivy_cve_list}" "${local_trivy_cve_list}") + if [[ -z "${cve_in_remote_but_not_in_local}" ]]; then + echo "Abort release. Fixed no new vulnerabilities" + exit 3 + fi + + build/make/release.sh "dogu-cve-release" "${cve_in_remote_but_not_in_local}" +} -USERNAME="${1}" -PASSWORD="${2}" - -if [ -z "${USERNAME}" ]; then - echo "username is unset" - while [[ -z ${USERNAME} ]]; do - read -r -p "type username for ${REGISTRY_URL}: " USERNAME - done -fi -if [ -z "${PASSWORD}" ]; then - echo "password is unset" - while [[ -z ${PASSWORD} ]]; do - read -r -s -p "type password for ${REGISTRY_URL}: " PASSWORD - done -fi - -dockerLogin "${USERNAME}" "${PASSWORD}" - -mkdir -p "${TRIVY_PATH}" # Cache will not be removed after release. rm requires root because the trivy container only runs with root. -pullRemoteImage -scanImage -REMOTE_TRIVY_CVE_LIST=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") - -buildLocalImage -scanImage -LOCAL_TRIVY_CVE_LIST=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") - -dockerLogout - -CVE_IN_REMOTE_BUT_NOT_LOCAL=$(diffArrays "${REMOTE_TRIVY_CVE_LIST}" "${LOCAL_TRIVY_CVE_LIST}") -CVE_IN_LOCAL_BUT_NOT_REMOTE=$(diffArrays "${LOCAL_TRIVY_CVE_LIST}" "${REMOTE_TRIVY_CVE_LIST}") - -if [[ -n "${CVE_IN_LOCAL_BUT_NOT_REMOTE}" ]]; then - echo "Abort release. Added new vulnerabilities:" - echo "${CVE_IN_LOCAL_BUT_NOT_REMOTE[@]}" - exit 2 -fi - -if [[ -z "${CVE_IN_REMOTE_BUT_NOT_LOCAL}" ]]; then - echo "Abort release. Fixed no new vulnerabilities" - exit 3 +# make the script only run when executed, not when sourced from bats tests +if [[ -n "${BASH_VERSION}" && "${BASH_SOURCE[0]}" == "${0}" ]]; then + USERNAME="${1}" + PASSWORD="${2}" + TRIVY_IMAGE_SCAN_FLAGS="${3:-""}" + run fi - -build/make/release.sh "dogu-cve-release" "${CVE_IN_REMOTE_BUT_NOT_LOCAL}" diff --git a/build/make/release_functions.sh b/build/make/release_functions.sh index 43501c0..921a7e5 100755 --- a/build/make/release_functions.sh +++ b/build/make/release_functions.sh @@ -5,8 +5,8 @@ set -o pipefail wait_for_ok() { printf "\n" - local OK=false - while [[ ${OK} != "ok" ]]; do + local OK="false" + while [[ "${OK}" != "ok" ]]; do read -r -p "${1} (type 'ok'): " OK done } @@ -174,7 +174,6 @@ update_changelog() { done if [[ -n "${FIXED_CVE_LIST}" ]]; then - # Add fixed CVEs in unreleased section addFixedCVEListFromReRelease "${FIXED_CVE_LIST}" fi @@ -198,29 +197,29 @@ update_changelog() { # addFixedCVEListFromReRelease is used in dogu cve releases. The method adds the fixed CVEs under the ### Fixed header # in the unreleased section. addFixedCVEListFromReRelease() { - local FIXED_CVE_LIST="${1}" + local fixed_cve_list="${1}" - local CVE_SED_SEARCH="" - local CVE_SED_REPLACE="" - local FIXED_EXISTS_IN_UNRELEASED - FIXED_EXISTS_IN_UNRELEASED=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^### Fixed$" || true) - if [[ -n "${FIXED_EXISTS_IN_UNRELEASED}" ]]; then + local cve_sed_search="" + local cve_sed_replace="" + local fixed_exists_in_unreleased + fixed_exists_in_unreleased=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^### Fixed$" || true) + if [[ -n "${fixed_exists_in_unreleased}" ]]; then # extend fixed header with CVEs. - CVE_SED_SEARCH="^\#\#\# Fixed$" - CVE_SED_REPLACE="\#\#\# Fixed\n- Fixed ${FIXED_CVE_LIST}" + cve_sed_search="^\#\#\# Fixed$" + cve_sed_replace="\#\#\# Fixed\n- Fixed ${fixed_cve_list}" else # extend unreleased header with fixed header and CVEs. - CVE_SED_SEARCH="^\#\# \[Unreleased\]$" - CVE_SED_REPLACE="\#\# \[Unreleased\]\n\#\#\# Fixed\n- Fixed ${FIXED_CVE_LIST}" + cve_sed_search="^\#\# \[Unreleased\]$" + cve_sed_replace="\#\# \[Unreleased\]\n\#\#\# Fixed\n- Fixed ${fixed_cve_list}" - local ANY_EXISTS_UNRELEASED - ANY_EXISTS_UNRELEASED=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^\#\#\# Added$" -e "^\#\#\# Fixed$" -e "^\#\#\# Changed$" || true) - if [[ -n ${ANY_EXISTS_UNRELEASED} ]]; then - CVE_SED_REPLACE+="\n" + local any_exists_unreleased + any_exists_unreleased=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^\#\#\# Added$" -e "^\#\#\# Fixed$" -e "^\#\#\# Changed$" || true) + if [[ -n ${any_exists_unreleased} ]]; then + cve_sed_replace+="\n" fi fi - sed -i "0,/${CVE_SED_SEARCH}/s//${CVE_SED_REPLACE}/" CHANGELOG.md + sed -i "0,/${cve_sed_search}/s//${cve_sed_replace}/" CHANGELOG.md } show_diff() { From b2a1e3491bf85e8bbf9fdc5229094100b65f10a5 Mon Sep 17 00:00:00 2001 From: Philipp Pixel Date: Thu, 14 Sep 2023 17:40:40 +0200 Subject: [PATCH 10/17] Test WIP, does not work out :( --- batsTests/dogu.json | 6 ++++++ batsTests/release_cve.bats | 15 +++++++++++++++ build/make/bats/Dockerfile | 13 ++++++++++++- build/make/k8s.mk | 3 +++ 4 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 batsTests/dogu.json diff --git a/batsTests/dogu.json b/batsTests/dogu.json new file mode 100644 index 0000000..b6a17f7 --- /dev/null +++ b/batsTests/dogu.json @@ -0,0 +1,6 @@ +{ + "Name": "official/le-dogu", + "Version": "1.2.3-4", + "Image": "registry.cloudogu.com/official/le-dogu" +} + diff --git a/batsTests/release_cve.bats b/batsTests/release_cve.bats index 8fba18a..d94ea31 100644 --- a/batsTests/release_cve.bats +++ b/batsTests/release_cve.bats @@ -13,11 +13,14 @@ setup() { export WORKDIR=/workspace export MAKE_DIR="${WORKDIR}"/build/make export PATH="${PATH}:${BATS_TMPDIR}" + docker="$(mock_create)" + ln -s "${docker}" "${BATS_TMPDIR}/docker" } teardown() { unset MAKE_DIR unset WORKDIR + rm "${BATS_TMPDIR}/docker" } @test "source script with bash should return exit code 0" { @@ -26,3 +29,15 @@ teardown() { assert_success } +@test "asdf 0" { + # shellcheck source=./../build/make/release_cve.sh + source "${MAKE_DIR}/release_cve.sh" + export USERNAME=Larry + export PASSWORD="ken sent me" + export DOGU_JSON_FILE=${WORKDIR}/ + + run diffArrays "CVE-11111 CVE-22222" "CVE-22222 CVE-33333" + + assert_success + assert_line "asdf" +} diff --git a/build/make/bats/Dockerfile b/build/make/bats/Dockerfile index f75afe1..c066fa8 100644 --- a/build/make/bats/Dockerfile +++ b/build/make/bats/Dockerfile @@ -2,6 +2,17 @@ ARG BATS_BASE_IMAGE ARG BATS_TAG FROM ${BATS_BASE_IMAGE}:${BATS_TAG} +ARG YQ_VERSION=v4.35.1 +ARG YQ_SHA256=bd695a6513f1196aeda17b174a15e9c351843fb1cef5f9be0af170f2dd744f08 # Make bash more findable by scripts and tests -RUN apk add make git bash \ No newline at end of file +RUN apk add coreutils make git bash jq wget +RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /tmp/yq && \ + echo "${YQ_SHA256} */tmp/yq" | sha256sum -c - && \ + mkdir -p /workspace/.bin/ && \ + mv /tmp/yq /workspace/.bin/ + +RUN addgroup -S -g 1000 bats \ + && adduser -S -h /home/bats -s /bin/bash -G bats -u 1000 bats + +USER 1000 diff --git a/build/make/k8s.mk b/build/make/k8s.mk index 5ea6750..9f0cac6 100644 --- a/build/make/k8s.mk +++ b/build/make/k8s.mk @@ -119,5 +119,8 @@ __check_defined = \ $(if $(value $1),, \ $(error Undefined $1$(if $2, ($2)))) +.PHONY: install-yq ## Installs the yq YAML editor. +install-yq: ${BINARY_YQ} + ${BINARY_YQ}: $(UTILITY_BIN_PATH) ## Download yq locally if necessary. $(call go-get-tool,$(BINARY_YQ),github.com/mikefarah/yq/v4@v4.25.1) From 35c86333db9c2bc3e157b8c3136b4302bad0b295 Mon Sep 17 00:00:00 2001 From: Niklas Date: Fri, 15 Sep 2023 11:18:23 +0200 Subject: [PATCH 11/17] #143 Add bats test for cve pipeline Co-authored-by: Philipp Pixel --- batsTests/release_cve.bats | 276 +++++++++++++++++++++++++++++++++++-- build/make/bats/Dockerfile | 13 +- build/make/release_cve.sh | 33 +++-- 3 files changed, 287 insertions(+), 35 deletions(-) diff --git a/batsTests/release_cve.bats b/batsTests/release_cve.bats index d94ea31..968e84d 100644 --- a/batsTests/release_cve.bats +++ b/batsTests/release_cve.bats @@ -12,15 +12,24 @@ load '/workspace/target/bats_libs/bats-file/load.bash' setup() { export WORKDIR=/workspace export MAKE_DIR="${WORKDIR}"/build/make - export PATH="${PATH}:${BATS_TMPDIR}" + export PATH="${BATS_TMPDIR}:${PATH}" docker="$(mock_create)" ln -s "${docker}" "${BATS_TMPDIR}/docker" + jq="$(mock_create)" + ln -s "${jq}" "${BATS_TMPDIR}/jq" + read="$(mock_create)" + ln -s "${read}" "${BATS_TMPDIR}/read" + release_script="$(mock_create)" + ln -s "${release_script}" "${BATS_TMPDIR}/release" } teardown() { unset MAKE_DIR unset WORKDIR - rm "${BATS_TMPDIR}/docker" + rm "${BATS_TMPDIR}/docker" + rm "${BATS_TMPDIR}/jq" + rm "${BATS_TMPDIR}/read" + rm "${BATS_TMPDIR}/release" } @test "source script with bash should return exit code 0" { @@ -29,15 +38,262 @@ teardown() { assert_success } -@test "asdf 0" { - # shellcheck source=./../build/make/release_cve.sh +@test "diffArrays should print values which are in the first but not in the second array" { source "${MAKE_DIR}/release_cve.sh" - export USERNAME=Larry - export PASSWORD="ken sent me" - export DOGU_JSON_FILE=${WORKDIR}/ - run diffArrays "CVE-11111 CVE-22222" "CVE-22222 CVE-33333" - + run diffArrays "CVE-11111 CVE-22222 CVE-33333" "CVE-22222" + + assert_success + assert_line "CVE-11111 CVE-33333" +} + +@test "diffArrays should print nothing on equal arrays" { + source "${MAKE_DIR}/release_cve.sh" + + local result + result=$(diffArrays "CVE-11111 CVE-22222 CVE-33333" "CVE-11111 CVE-22222 CVE-33333") + + assert_equal "${result}" "" +} + +@test "diffArrays should print nothing on empty arrays" { + source "${MAKE_DIR}/release_cve.sh" + + local result + result=$(diffArrays "" "") + + assert_equal "${result}" "" +} + +@test "docker login should call the login sub command with provided globals" { + source "${MAKE_DIR}/release_cve.sh" + + export USERNAME="user" + export PASSWORD="password" + export REGISTRY_URL="registry" + + run dockerLogin + + assert_success + assert_equal "$(mock_get_call_num "${docker}")" "1" + assert_equal "$(mock_get_call_args "${docker}" "1")" "login registry -u user -p password" +} + +@test "docker logout should call the logout sub command with registry globals" { + source "${MAKE_DIR}/release_cve.sh" + + export REGISTRY_URL="registry" + + run dockerLogout + + assert_success + assert_equal "$(mock_get_call_num "${docker}")" "1" + assert_equal "$(mock_get_call_args "${docker}" "1")" "logout registry" +} + +@test "nameFromDogu should return the name from the dogu.json file" { + source "${MAKE_DIR}/release_cve.sh" + export DOGU_JSON_FILE="dogu.json" + + run nameFromDogu + + assert_success + assert_equal "$(mock_get_call_num "${jq}")" "1" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Name ${DOGU_JSON_FILE}" +} + +@test "versionFromDogu should return the name from the dogu.json file" { + source "${MAKE_DIR}/release_cve.sh" + export DOGU_JSON_FILE="dogu.json" + + run versionFromDogu + + assert_success + assert_equal "$(mock_get_call_num "${jq}")" "1" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Version ${DOGU_JSON_FILE}" +} + +@test "imageFromDogu should return the name from the dogu.json file" { + source "${MAKE_DIR}/release_cve.sh" + export DOGU_JSON_FILE="dogu.json" + + run imageFromDogu + + assert_success + assert_equal "$(mock_get_call_num "${jq}")" "1" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Image ${DOGU_JSON_FILE}" +} + +@test "jsonPropertyFromDogu should call jq with the dogu.json file global variable and the provided parameter" { + source "${MAKE_DIR}/release_cve.sh" + export DOGU_JSON_FILE="dogu.json" + + run jsonPropertyFromDogu "property" + + assert_success + assert_equal "$(mock_get_call_num "${jq}")" "1" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r property ${DOGU_JSON_FILE}" +} + +@test "readCredentialsIfUnset should not ask to enter credentials if they are set" { + source "${MAKE_DIR}/release_cve.sh" + export USERNAME="user" + export PASSWORD="password" + + run readCredentialsIfUnset + + assert_success + assert_equal "$(mock_get_call_num "${read}")" "0" +} + +@test "parseTrivyJsonResult should call jq with given severity and result file" { + source "${MAKE_DIR}/release_cve.sh" + export PASSWORD="password" + + run parseTrivyJsonResult "severity" "result.json" + + assert_success + assert_equal "$(mock_get_call_num "${jq}")" "1" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"severity\") | .VulnerabilityID] | join(\" \") result.json" +} + +@test "runMain should not start release process if cve were added" { + source "${MAKE_DIR}/release_cve.sh" + export TRIVY_PATH="${BATS_TMPDIR}/trivy" + export TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + export TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + export TRIVY_DOCKER_CACHE_DIR=/tmp/db + export TRIVY_IMAGE_SCAN_FLAGS="--use this" + + export USERNAME="user" + export PASSWORD="password" + + mock_set_output "${jq}" "jenkins" "1" + mock_set_output "${jq}" "1.0.0" "2" + mock_set_output "${jq}" "jenkins" "3" + mock_set_output "${jq}" "1.0.0" "4" + mock_set_output "${jq}" "CVE-1" "5" + mock_set_output "${jq}" "jenkins" "6" + mock_set_output "${jq}" "1.0.0" "7" + mock_set_output "${jq}" "jenkins" "8" + mock_set_output "${jq}" "1.0.0" "9" + mock_set_output "${jq}" "CVE-1 CVE-2" "10" + + run runMain + + assert_equal "$(mock_get_call_num "${docker}")" "6" + assert_equal "$(mock_get_call_args "${docker}" "1")" "login registry.cloudogu.com -u user -p password" + assert_equal "$(mock_get_call_args "${docker}" "2")" "pull jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "3")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "4")" "build . -t jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "5")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "6")" "logout registry.cloudogu.com" + assert_equal "$(mock_get_call_num "${jq}")" "10" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "2")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "3")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "4")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "5")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_args "${jq}" "6")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "7")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "8")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "9")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "10")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_line "Abort release. Added new vulnerabilities:" + assert_line "CVE-2" + assert_failure "2" +} + +@test "runMain should not start release process if no cve will be fixed" { + source "${MAKE_DIR}/release_cve.sh" + export TRIVY_PATH="${BATS_TMPDIR}/trivy" + export TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + export TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + export TRIVY_DOCKER_CACHE_DIR=/tmp/db + export TRIVY_IMAGE_SCAN_FLAGS="--use this" + + export USERNAME="user" + export PASSWORD="password" + + mock_set_output "${jq}" "jenkins" "1" + mock_set_output "${jq}" "1.0.0" "2" + mock_set_output "${jq}" "jenkins" "3" + mock_set_output "${jq}" "1.0.0" "4" + mock_set_output "${jq}" "CVE-1 CVE-2" "5" + mock_set_output "${jq}" "jenkins" "6" + mock_set_output "${jq}" "1.0.0" "7" + mock_set_output "${jq}" "jenkins" "8" + mock_set_output "${jq}" "1.0.0" "9" + mock_set_output "${jq}" "CVE-1 CVE-2" "10" + + run runMain + + assert_equal "$(mock_get_call_num "${docker}")" "6" + assert_equal "$(mock_get_call_args "${docker}" "1")" "login registry.cloudogu.com -u user -p password" + assert_equal "$(mock_get_call_args "${docker}" "2")" "pull jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "3")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "4")" "build . -t jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "5")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "6")" "logout registry.cloudogu.com" + assert_equal "$(mock_get_call_num "${jq}")" "10" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "2")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "3")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "4")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "5")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_args "${jq}" "6")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "7")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "8")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "9")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "10")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_line "Abort release. Fixed no new vulnerabilities" + assert_failure "3" +} + +@test "runMain should start release process if cves will be fixed" { + source "${MAKE_DIR}/release_cve.sh" + export TRIVY_PATH="${BATS_TMPDIR}/trivy" + export TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + export TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + export TRIVY_DOCKER_CACHE_DIR=/tmp/db + export TRIVY_IMAGE_SCAN_FLAGS="--use this" + export RELEASE_SH="${release_script}" + + export USERNAME="user" + export PASSWORD="password" + + mock_set_output "${jq}" "jenkins" "1" + mock_set_output "${jq}" "1.0.0" "2" + mock_set_output "${jq}" "jenkins" "3" + mock_set_output "${jq}" "1.0.0" "4" + mock_set_output "${jq}" "CVE-1 CVE-2" "5" + mock_set_output "${jq}" "jenkins" "6" + mock_set_output "${jq}" "1.0.0" "7" + mock_set_output "${jq}" "jenkins" "8" + mock_set_output "${jq}" "1.0.0" "9" + mock_set_output "${jq}" "CVE-1" "10" + + run runMain + + assert_equal "$(mock_get_call_num "${docker}")" "6" + assert_equal "$(mock_get_call_args "${docker}" "1")" "login registry.cloudogu.com -u user -p password" + assert_equal "$(mock_get_call_args "${docker}" "2")" "pull jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "3")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "4")" "build . -t jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "5")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "6")" "logout registry.cloudogu.com" + assert_equal "$(mock_get_call_num "${jq}")" "10" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "2")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "3")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "4")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "5")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_args "${jq}" "6")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "7")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "8")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "9")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "10")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_num "${release_script}")" "1" + assert_equal "$(mock_get_call_args "${release_script}" "1")" "dogu-cve-release CVE-2" assert_success - assert_line "asdf" } diff --git a/build/make/bats/Dockerfile b/build/make/bats/Dockerfile index c066fa8..428ee05 100644 --- a/build/make/bats/Dockerfile +++ b/build/make/bats/Dockerfile @@ -2,17 +2,6 @@ ARG BATS_BASE_IMAGE ARG BATS_TAG FROM ${BATS_BASE_IMAGE}:${BATS_TAG} -ARG YQ_VERSION=v4.35.1 -ARG YQ_SHA256=bd695a6513f1196aeda17b174a15e9c351843fb1cef5f9be0af170f2dd744f08 # Make bash more findable by scripts and tests -RUN apk add coreutils make git bash jq wget -RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /tmp/yq && \ - echo "${YQ_SHA256} */tmp/yq" | sha256sum -c - && \ - mkdir -p /workspace/.bin/ && \ - mv /tmp/yq /workspace/.bin/ - -RUN addgroup -S -g 1000 bats \ - && adduser -S -h /home/bats -s /bin/bash -G bats -u 1000 bats - -USER 1000 +RUN apk add make git bash diff --git a/build/make/release_cve.sh b/build/make/release_cve.sh index 8a248e6..1fafabc 100755 --- a/build/make/release_cve.sh +++ b/build/make/release_cve.sh @@ -26,10 +26,9 @@ function diffArrays() { local cveX # Disable the following shellcheck because the arrays are sufficiently whitespace delimited because of the jq parsing result. # shellcheck disable=SC2128 - for cveX in "${cveListX[@]}"; do + for cveX in ${cveListX}; do local found=0 local cveY - echo "$cveX" for cveY in ${cveListY}; do [[ "${cveY}" == "${cveX}" ]] && { found=1 @@ -59,7 +58,7 @@ function imageFromDogu() { jsonPropertyFromDogu ".Image" } -function versionsFromDogu() { +function versionFromDogu() { jsonPropertyFromDogu ".Version" } @@ -83,36 +82,41 @@ function scanImage() { function parseTrivyJsonResult() { local severity="${1}" local trivy_result_file="${2}" - local cve_result="" - cve_result=$(jq -rc "[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"${severity}\") | .VulnerabilityID] | join(\" \")" "${trivy_result_file}") - echo "${cve_result}" + + # First select results which have the property "Vulnerabilities". Filter the vulnerability ids with the given severity and afterward put the values in an array. + # This array is used to format the values with join(" ") in a whitespace delimited string list. + jq -rc "[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"${severity}\") | .VulnerabilityID] | join(\" \")" "${trivy_result_file}" } +RELEASE_SH="build/make/release.sh" + REGISTRY_URL="registry.cloudogu.com" DOGU_JSON_FILE="dogu.json" CVE_SEVERITY="CRITICAL" -TRIVY_PATH="/tmp/trivy-dogu-cve-release-$(nameFromDogu)" -TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" -TRIVY_CACHE_DIR="${TRIVY_PATH}/db" +TRIVY_PATH= +TRIVY_RESULT_FILE= +TRIVY_CACHE_DIR= TRIVY_DOCKER_CACHE_DIR=/tmp/db TRIVY_IMAGE_SCAN_FLAGS= USERNAME="" PASSWORD="" -function run() { +function runMain() { readCredentialsIfUnset dockerLogin mkdir -p "${TRIVY_PATH}" # Cache will not be removed after release. rm requires root because the trivy container only runs with root. pullRemoteImage scanImage + local remote_trivy_cve_list remote_trivy_cve_list=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") buildLocalImage scanImage + local local_trivy_cve_list local_trivy_cve_list=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") dockerLogout @@ -132,13 +136,16 @@ function run() { exit 3 fi - build/make/release.sh "dogu-cve-release" "${cve_in_remote_but_not_in_local}" + "${RELEASE_SH}" "dogu-cve-release" "${cve_in_remote_but_not_in_local}" } -# make the script only run when executed, not when sourced from bats tests +# make the script only runMain when executed, not when sourced from bats tests if [[ -n "${BASH_VERSION}" && "${BASH_SOURCE[0]}" == "${0}" ]]; then USERNAME="${1}" PASSWORD="${2}" TRIVY_IMAGE_SCAN_FLAGS="${3:-""}" - run + TRIVY_PATH="/tmp/trivy-dogu-cve-release-$(nameFromDogu)" + TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + runMain fi From c36055573cb34be76bea6dbfd710fa22fd59d0b7 Mon Sep 17 00:00:00 2001 From: Philipp Pixel Date: Fri, 15 Sep 2023 10:22:32 +0200 Subject: [PATCH 12/17] Remove errors by cleaning up actual makefile to run internal shell tests Originally, the main `Makefile` was intended for grabs. Now that the shell script complexity surges it is necessary to test these scripts in an automated way - of course with make itself! The current state of the Makefile produces lots of errors because exlusive targets overwrite themselves. This commit moves the original Makefile as a code snippet into the README.md and uses only necessary parts to run tests or provide an entry point for automatic release. --- Makefile | 53 ++------------------------------------------ README.md | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 51 deletions(-) diff --git a/Makefile b/Makefile index 795446e..05e86d6 100644 --- a/Makefile +++ b/Makefile @@ -1,61 +1,12 @@ # Set these to the desired values -ARTIFACT_ID= -VERSION= - +ARTIFACT_ID=makefiles MAKEFILES_VERSION=8.0.0 +VERSION=${MAKEFILES_VERSION} .DEFAULT_GOAL:=help -# set PRE_COMPILE to define steps that shall be executed before the go build -# PRE_COMPILE= - -# set GO_ENV_VARS to define go environment variables for the go build -# GO_ENV_VARS = CGO_ENABLED=0 - -# set PRE_UNITTESTS and POST_UNITTESTS to define steps that shall be executed before or after the unit tests -# PRE_UNITTESTS?= -# POST_UNITTESTS?= - -# set PREPARE_PACKAGE to define a target that should be executed before the package build -# PREPARE_PACKAGE= - -# set ADDITIONAL_CLEAN to define a target that should be executed before the clean target, e.g. -# ADDITIONAL_CLEAN=clean_deb -# clean_deb: -# rm -rf ${DEBIAN_BUILD_DIR} - -# APT_REPO controls the target apt repository for deploy-debian.mk -# -> APT_REPO=ces-premium results in a deploy to the premium apt repository -# -> Everything else results in a deploy to the public repositories -APT_REPO?=ces - include build/make/variables.mk - -# You may want to overwrite existing variables for target actions to fit into your project. - -include build/make/self-update.mk -include build/make/dependencies-gomod.mk -include build/make/build.mk -include build/make/test-common.mk -include build/make/test-integration.mk -include build/make/test-unit.mk -include build/make/mocks.mk -include build/make/static-analysis.mk include build/make/clean.mk -# either package-tar.mk -include build/make/package-tar.mk -# or package-debian.mk -include build/make/package-debian.mk -# deploy-debian.mk depends on package-debian.mk -include build/make/deploy-debian.mk include build/make/digital-signature.mk -include build/make/yarn.mk -include build/make/bower.mk -# only include this in repositories which support the automatic release process (like dogus or golang apps) include build/make/release.mk -# either k8s-dogu.mk -include build/make/k8s-dogu.mk -# or k8s-controller.mk; only include this in k8s-controller repositories -include build/make/k8s-controller.mk include build/make/bats.mk - diff --git a/README.md b/README.md index 04aec02..f618852 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,72 @@ This repository holds makefiles for building Cloudogu tools, especially those wr Please note that `make` only accepts `Makefile`s that are **only** indented with tabs. +## Quick Example + +```makefile +# Set these to the desired values +ARTIFACT_ID= +VERSION= + +MAKEFILES_VERSION=8.0.0 + +.DEFAULT_GOAL:=help + +# set PRE_COMPILE to define steps that shall be executed before the go build +# PRE_COMPILE= + +# set GO_ENV_VARS to define go environment variables for the go build +# GO_ENV_VARS = CGO_ENABLED=0 + +# set PRE_UNITTESTS and POST_UNITTESTS to define steps that shall be executed before or after the unit tests +# PRE_UNITTESTS?= +# POST_UNITTESTS?= + +# set PREPARE_PACKAGE to define a target that should be executed before the package build +# PREPARE_PACKAGE= + +# set ADDITIONAL_CLEAN to define a target that should be executed before the clean target, e.g. +# ADDITIONAL_CLEAN=clean_deb +# clean_deb: +# rm -rf ${DEBIAN_BUILD_DIR} + +# APT_REPO controls the target apt repository for deploy-debian.mk +# -> APT_REPO=ces-premium results in a deploy to the premium apt repository +# -> Everything else results in a deploy to the public repositories +APT_REPO?=ces + +include build/make/variables.mk + +# You may want to overwrite existing variables for target actions to fit into your project. + +include build/make/self-update.mk +include build/make/dependencies-gomod.mk +include build/make/build.mk +include build/make/test-common.mk +include build/make/test-integration.mk +include build/make/test-unit.mk +include build/make/mocks.mk +include build/make/static-analysis.mk +include build/make/clean.mk +# either package-tar.mk +include build/make/package-tar.mk +# or package-debian.mk +include build/make/package-debian.mk +# deploy-debian.mk depends on package-debian.mk +include build/make/deploy-debian.mk +include build/make/digital-signature.mk +include build/make/yarn.mk +include build/make/bower.mk +# only include this in repositories which support the automatic release process (like dogus or golang apps) +include build/make/release.mk +# either k8s-dogu.mk +include build/make/k8s-dogu.mk +# or k8s-controller.mk; only include this in k8s-controller repositories +include build/make/k8s-controller.mk +include build/make/bats.mk + +``` + ## Overview over make targets Starting with makefiles v5.0.0 `make help` will produce an overview of make popular targets: From af99e3b0863dabca35c39981f613c2a3dfc7a93f Mon Sep 17 00:00:00 2001 From: Philipp Pixel Date: Fri, 15 Sep 2023 11:04:05 +0200 Subject: [PATCH 13/17] add missing k8s targets to the README.md --- README.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/README.md b/README.md index f618852..94b0d67 100644 --- a/README.md +++ b/README.md @@ -354,3 +354,62 @@ Additionally, to the regular `dogu-release` the module contains a `dogu-cve-rele build of a dogu eliminates critical CVEs. If this is the case, a release process will be triggered. Only include this module in dogu or Golang repositories that support a dedicated release flow! +### bats.mk + +This module enables you to run BATS shell tests via the `unit-test-shell` target. All you need is a directory with BATS +tests in `${yourProjectDir}/batsTEsts` (overrideable with the variable `TESTS_DIR`). + +### K8s-related makefiles + +#### k8s.mk + +This module provides generic targets for developing K8s Cloudogu EcoSystem + +- `image-import` imports the currently available image into the cluster-local registry. +- `docker-dev-tag` tags a Docker image for local K8s-CES deployment. +- `docker-build` builds the docker image of the K8s app. +- `k8s-generate` generates one concatenated resource YAML +- `k8s-apply` applies all generated K8s resources to the current cluster and namespace +- check single or all of these variables: + - `check-all-vars` + - `check-k8s-namespace-env-var` + - `check-k8s-image-env-var` + - `check-k8s-artifact-id` + - `check-etc-hosts` + - `check-insecure-cluster-registry` + +#### k8s-component.mk + +This module provides targets for developing K8s Cloudogu EcoSystem components (including controllers) +- General helm targets + - `helm-init-chart` - Creates a Chart.yaml-template with zero values + - `helm-generate-chart` - Generates the final helm chart +- Helm developing targets + - `helm-generate` - Generates the final helm chart with dev-urls + - `helm-apply` - Generates and installs the helm chart + - `helm-delete` - Uninstalls the current helm chart + - `helm-reinstall` - Uninstalls the current helm chart and re-installs it + - `helm-chart-import` - Imports the currently available chart into the cluster-local registry +- Release targets + - `helm-package-release` - Generates and packages the helm chart with release urls. + - `helm-generate-release` - Generates the final helm chart with release urls. +- Component-oriented targets + - `component-generate` - Generate the component YAML resource + - `component-apply` - Applies the component yaml resource to the actual defined context. + - `component-reinstall` - Reinstalls the component yaml resource from the actual defined context. + - `component-delete` - Deletes the component yaml resource from the actual defined context. + +#### k8s-dogu.mk + +This module provides targets for developing Dogus with a K8s Cloudogu EcoSystem. + +- `build` - Builds a new version of the dogu and deploys it into the K8s-EcoSystem. +- `install-dogu-descriptor` - Installs a configmap with current dogu.json into the cluster. + +#### k8s-controller.mk + +This module provides targets for K8s Cloudogu EcoSystem controllers. + +- `k8s-integration-test` - Run k8s integration tests. +- `controller-release` - Interactively starts the release workflow. +- `build: helm-apply` - Builds a new version of the dogu and deploys it into the K8s-EcoSystem. From ff89503f44ea5fce27e3ccdbc25eddf4667c5a07 Mon Sep 17 00:00:00 2001 From: Philipp Pixel Date: Fri, 15 Sep 2023 13:00:44 +0200 Subject: [PATCH 14/17] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68c9e7d..72f3347 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - [#143] Add release target `dogu-cve-release` for dogus if a simple rebuild fixes critical CVEs. +- Add missing K8s and bats target descriptions on the [README.md](README.md) ## [v8.0.0](https://github.com/cloudogu/makefiles/releases/tag/v8.0.0) 2023-09-12 ### Added From f6a60052219ad7ef48bf90a99ace270b5b8f8781 Mon Sep 17 00:00:00 2001 From: Niklas Date: Fri, 15 Sep 2023 14:08:42 +0200 Subject: [PATCH 15/17] #143 Add dry run Co-authored-by: Philipp Pixel --- CHANGELOG.md | 1 + batsTests/dogu.json | 6 ---- batsTests/release_cve.bats | 53 +++++++++++++++++++++++++++++++-- build/make/k8s.mk | 2 +- build/make/release.mk | 2 +- build/make/release.sh | 14 +++++++-- build/make/release_cve.sh | 11 ++++--- build/make/release_functions.sh | 13 ++++++++ 8 files changed, 86 insertions(+), 16 deletions(-) delete mode 100644 batsTests/dogu.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 72f3347..4ad3060 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - [#143] Add release target `dogu-cve-release` for dogus if a simple rebuild fixes critical CVEs. + - The target can be executed with a `DRY_RUN` environment variable for added developer experience. - Add missing K8s and bats target descriptions on the [README.md](README.md) ## [v8.0.0](https://github.com/cloudogu/makefiles/releases/tag/v8.0.0) 2023-09-12 diff --git a/batsTests/dogu.json b/batsTests/dogu.json deleted file mode 100644 index b6a17f7..0000000 --- a/batsTests/dogu.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "Name": "official/le-dogu", - "Version": "1.2.3-4", - "Image": "registry.cloudogu.com/official/le-dogu" -} - diff --git a/batsTests/release_cve.bats b/batsTests/release_cve.bats index 968e84d..a8c8e17 100644 --- a/batsTests/release_cve.bats +++ b/batsTests/release_cve.bats @@ -250,7 +250,7 @@ teardown() { assert_failure "3" } -@test "runMain should start release process if cves will be fixed" { +@test "runMain should start release process if cves will be fixed without dry run option" { source "${MAKE_DIR}/release_cve.sh" export TRIVY_PATH="${BATS_TMPDIR}/trivy" export TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" @@ -294,6 +294,55 @@ teardown() { assert_equal "$(mock_get_call_args "${jq}" "9")" "-r .Version dogu.json" assert_equal "$(mock_get_call_args "${jq}" "10")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" assert_equal "$(mock_get_call_num "${release_script}")" "1" - assert_equal "$(mock_get_call_args "${release_script}" "1")" "dogu-cve-release CVE-2" + assert_equal "$(mock_get_call_args "${release_script}" "1")" "dogu-cve-release CVE-2 " + assert_success +} + +@test "runMain should start release process if cves will be fixed with dry run option" { + source "${MAKE_DIR}/release_cve.sh" + export TRIVY_PATH="${BATS_TMPDIR}/trivy" + export TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + export TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + export TRIVY_DOCKER_CACHE_DIR=/tmp/db + export TRIVY_IMAGE_SCAN_FLAGS="--use this" + export RELEASE_SH="${release_script}" + export DRY_RUN="true" + + export USERNAME="user" + export PASSWORD="password" + + mock_set_output "${jq}" "jenkins" "1" + mock_set_output "${jq}" "1.0.0" "2" + mock_set_output "${jq}" "jenkins" "3" + mock_set_output "${jq}" "1.0.0" "4" + mock_set_output "${jq}" "CVE-1 CVE-2" "5" + mock_set_output "${jq}" "jenkins" "6" + mock_set_output "${jq}" "1.0.0" "7" + mock_set_output "${jq}" "jenkins" "8" + mock_set_output "${jq}" "1.0.0" "9" + mock_set_output "${jq}" "CVE-1" "10" + + run runMain + + assert_equal "$(mock_get_call_num "${docker}")" "6" + assert_equal "$(mock_get_call_args "${docker}" "1")" "login registry.cloudogu.com -u user -p password" + assert_equal "$(mock_get_call_args "${docker}" "2")" "pull jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "3")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "4")" "build . -t jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "5")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "6")" "logout registry.cloudogu.com" + assert_equal "$(mock_get_call_num "${jq}")" "10" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "2")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "3")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "4")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "5")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_args "${jq}" "6")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "7")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "8")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "9")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "10")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_num "${release_script}")" "1" + assert_equal "$(mock_get_call_args "${release_script}" "1")" "dogu-cve-release CVE-2 true" assert_success } diff --git a/build/make/k8s.mk b/build/make/k8s.mk index 9f0cac6..3f2ab94 100644 --- a/build/make/k8s.mk +++ b/build/make/k8s.mk @@ -67,7 +67,7 @@ K8S_PRE_GENERATE_TARGETS ?= k8s-create-temporary-resource k8s-generate: ${BINARY_YQ} $(K8S_RESOURCE_TEMP_FOLDER) $(K8S_PRE_GENERATE_TARGETS) ## Generates the final resource yaml. @echo "Applying general transformations..." @sed -i "s/'{{ .Namespace }}'/$(NAMESPACE)/" $(K8S_RESOURCE_TEMP_YAML) - @if [[ ${STAGE} == "development" ]]; then \ + @if [[ "${STAGE}" == "development" ]]; then \ $(BINARY_YQ) -i e "(select(.kind == \"Deployment\").spec.template.spec.containers[]|select(.image == \"*$(ARTIFACT_ID)*\").image)=\"$(IMAGE_DEV)\"" $(K8S_RESOURCE_TEMP_YAML); \ else \ $(BINARY_YQ) -i e "(select(.kind == \"Deployment\").spec.template.spec.containers[]|select(.image == \"*$(ARTIFACT_ID)*\").image)=\"$(IMAGE)\"" $(K8S_RESOURCE_TEMP_YAML); \ diff --git a/build/make/release.mk b/build/make/release.mk index 99da81c..82ba1ba 100644 --- a/build/make/release.mk +++ b/build/make/release.mk @@ -12,4 +12,4 @@ go-release: ## Start a go tool release .PHONY: dogu-cve-release dogu-cve-release: ## Start a dogu release of a new build if the local build fixes critical CVEs - @bash -c "build/make/release_cve.sh ${REGISTRY_USERNAME} ${REGISTRY_PASSWORD}" \ No newline at end of file + @bash -c "build/make/release_cve.sh \"${REGISTRY_USERNAME}\" \"${REGISTRY_PASSWORD}\" \"${TRIVY_IMAGE_SCAN_FLAGS}\" \"${DRY_RUN}\"" diff --git a/build/make/release.sh b/build/make/release.sh index c0eef3f..ae9a722 100755 --- a/build/make/release.sh +++ b/build/make/release.sh @@ -35,6 +35,7 @@ source "$(pwd)/build/make/release_functions.sh" TYPE="${1}" FIXED_CVE_LIST="${2:-""}" +DRY_RUN="${3:-""}" echo "=====Starting Release process=====" @@ -47,11 +48,20 @@ fi NEW_RELEASE_VERSION="$(read_new_version)" validate_new_version "${NEW_RELEASE_VERSION}" -start_git_flow_release "${NEW_RELEASE_VERSION}" +if [[ -n "${DRY_RUN}" ]]; then + start_dry_run_release "${NEW_RELEASE_VERSION}" +else + start_git_flow_release "${NEW_RELEASE_VERSION}" +fi + update_versions "${NEW_RELEASE_VERSION}" update_changelog "${NEW_RELEASE_VERSION}" "${FIXED_CVE_LIST}" show_diff -finish_release_and_push "${CURRENT_TOOL_VERSION}" "${NEW_RELEASE_VERSION}" +if [[ -n "${DRY_RUN}" ]]; then + abort_dry_run_release "${NEW_RELEASE_VERSION}" +else + finish_release_and_push "${CURRENT_TOOL_VERSION}" "${NEW_RELEASE_VERSION}" +fi echo "=====Finished Release process=====" diff --git a/build/make/release_cve.sh b/build/make/release_cve.sh index 1fafabc..21738cb 100755 --- a/build/make/release_cve.sh +++ b/build/make/release_cve.sh @@ -76,7 +76,7 @@ function buildLocalImage() { } function scanImage() { - docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image "${TRIVY_IMAGE_SCAN_FLAGS}" "$(imageFromDogu):$(versionFromDogu)" + docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS:+"${TRIVY_IMAGE_SCAN_FLAGS}"} "$(imageFromDogu):$(versionFromDogu)" } function parseTrivyJsonResult() { @@ -103,6 +103,7 @@ TRIVY_IMAGE_SCAN_FLAGS= USERNAME="" PASSWORD="" +DRY_RUN= function runMain() { readCredentialsIfUnset @@ -136,14 +137,16 @@ function runMain() { exit 3 fi - "${RELEASE_SH}" "dogu-cve-release" "${cve_in_remote_but_not_in_local}" + "${RELEASE_SH}" "dogu-cve-release" "${cve_in_remote_but_not_in_local}" "${DRY_RUN}" } # make the script only runMain when executed, not when sourced from bats tests if [[ -n "${BASH_VERSION}" && "${BASH_SOURCE[0]}" == "${0}" ]]; then - USERNAME="${1}" - PASSWORD="${2}" + USERNAME="${1:-""}" + PASSWORD="${2:-""}" TRIVY_IMAGE_SCAN_FLAGS="${3:-""}" + DRY_RUN="${4:-""}" + TRIVY_PATH="/tmp/trivy-dogu-cve-release-$(nameFromDogu)" TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" TRIVY_CACHE_DIR="${TRIVY_PATH}/db" diff --git a/build/make/release_functions.sh b/build/make/release_functions.sh index 921a7e5..499c248 100755 --- a/build/make/release_functions.sh +++ b/build/make/release_functions.sh @@ -73,6 +73,19 @@ start_git_flow_release() { git flow release start v"${NEW_RELEASE_VERSION}" } +start_dry_run_release() { + local NEW_RELEASE_VERSION="${1}" + + git checkout -b dryrun/v"${NEW_RELEASE_VERSION}" +} + +abort_dry_run_release() { + local NEW_RELEASE_VERSION="${1}" + + git checkout develop + git branch -D dryrun/v"${NEW_RELEASE_VERSION}" +} + # update_versions updates files with the new release version and interactively asks the user for verification. If okay # the updated files will be staged to git and finally committed. # From 4b048609e637a008f16057f17481920ed1898fc4 Mon Sep 17 00:00:00 2001 From: Philipp Pixel Date: Fri, 15 Sep 2023 14:22:37 +0200 Subject: [PATCH 16/17] Format bats-targets, fix typo in README.md Co-authored-by: nhinze23 --- README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 94b0d67..d8c2a03 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,6 @@ include build/make/k8s-dogu.mk # or k8s-controller.mk; only include this in k8s-controller repositories include build/make/k8s-controller.mk include build/make/bats.mk - ``` ## Overview over make targets @@ -357,7 +356,7 @@ Only include this module in dogu or Golang repositories that support a dedicated ### bats.mk This module enables you to run BATS shell tests via the `unit-test-shell` target. All you need is a directory with BATS -tests in `${yourProjectDir}/batsTEsts` (overrideable with the variable `TESTS_DIR`). +tests in `${yourProjectDir}/batsTests` (overrideable with the variable `TESTS_DIR`). ### K8s-related makefiles @@ -365,11 +364,11 @@ tests in `${yourProjectDir}/batsTEsts` (overrideable with the variable `TESTS_DI This module provides generic targets for developing K8s Cloudogu EcoSystem -- `image-import` imports the currently available image into the cluster-local registry. -- `docker-dev-tag` tags a Docker image for local K8s-CES deployment. -- `docker-build` builds the docker image of the K8s app. -- `k8s-generate` generates one concatenated resource YAML -- `k8s-apply` applies all generated K8s resources to the current cluster and namespace +- `image-import` - imports the currently available image into the cluster-local registry. +- `docker-dev-tag` - tags a Docker image for local K8s-CES deployment. +- `docker-build` - builds the docker image of the K8s app. +- `k8s-generate` - generates one concatenated resource YAML +- `k8s-apply` - applies all generated K8s resources to the current cluster and namespace - check single or all of these variables: - `check-all-vars` - `check-k8s-namespace-env-var` @@ -396,7 +395,7 @@ This module provides targets for developing K8s Cloudogu EcoSystem components (i - Component-oriented targets - `component-generate` - Generate the component YAML resource - `component-apply` - Applies the component yaml resource to the actual defined context. - - `component-reinstall` - Reinstalls the component yaml resource from the actual defined context. + - `component-reinstall` - Re-installs the component yaml resource from the actual defined context. - `component-delete` - Deletes the component yaml resource from the actual defined context. #### k8s-dogu.mk From 46a9d52a15e24422f4d24c10dec009a4f757799f Mon Sep 17 00:00:00 2001 From: Philipp Pixel Date: Fri, 15 Sep 2023 14:31:32 +0200 Subject: [PATCH 17/17] Bump version v8.2.0 --- CHANGELOG.md | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f85e63..59cd256 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [v8.2.0](https://github.com/cloudogu/makefiles/releases/tag/v8.2.0) 2023-09-15 ### Added - [#143] Add release target `dogu-cve-release` for dogus if a simple rebuild fixes critical CVEs. - The target can be executed with a `DRY_RUN` environment variable for added developer experience. diff --git a/Makefile b/Makefile index 603e609..e0b213b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Set these to the desired values ARTIFACT_ID=makefiles -MAKEFILES_VERSION=8.1.0 +MAKEFILES_VERSION=8.2.0 VERSION=${MAKEFILES_VERSION} .DEFAULT_GOAL:=help