diff --git a/.gitignore b/.gitignore index 485dee6..6dcf23c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ .idea +target +.bin diff --git a/CHANGELOG.md b/CHANGELOG.md index 16bef57..59cd256 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [v8.2.0](https://github.com/cloudogu/makefiles/releases/tag/v8.2.0) 2023-09-15 +### Added +- [#143] Add release target `dogu-cve-release` for dogus if a simple rebuild fixes critical CVEs. + - The target can be executed with a `DRY_RUN` environment variable for added developer experience. +- Add missing K8s and bats target descriptions on the [README.md](README.md) + ## [v8.1.0](https://github.com/cloudogu/makefiles/releases/tag/v8.1.0) 2023-09-15 ### Removed - [#147] Remove Dummy-Chart-Dependencies from Helm-Chart-Generation diff --git a/Makefile b/Makefile index d6f9dbc..e0b213b 100644 --- a/Makefile +++ b/Makefile @@ -1,60 +1,12 @@ # Set these to the desired values -ARTIFACT_ID= -VERSION= - -MAKEFILES_VERSION=8.1.0 +ARTIFACT_ID=makefiles +MAKEFILES_VERSION=8.2.0 +VERSION=${MAKEFILES_VERSION} .DEFAULT_GOAL:=help -# set PRE_COMPILE to define steps that shall be executed before the go build -# PRE_COMPILE= - -# set GO_ENV_VARS to define go environment variables for the go build -# GO_ENV_VARS = CGO_ENABLED=0 - -# set PRE_UNITTESTS and POST_UNITTESTS to define steps that shall be executed before or after the unit tests -# PRE_UNITTESTS?= -# POST_UNITTESTS?= - -# set PREPARE_PACKAGE to define a target that should be executed before the package build -# PREPARE_PACKAGE= - -# set ADDITIONAL_CLEAN to define a target that should be executed before the clean target, e.g. -# ADDITIONAL_CLEAN=clean_deb -# clean_deb: -# rm -rf ${DEBIAN_BUILD_DIR} - -# APT_REPO controls the target apt repository for deploy-debian.mk -# -> APT_REPO=ces-premium results in a deploy to the premium apt repository -# -> Everything else results in a deploy to the public repositories -APT_REPO?=ces - include build/make/variables.mk - -# You may want to overwrite existing variables for target actions to fit into your project. - -include build/make/self-update.mk -include build/make/dependencies-gomod.mk -include build/make/build.mk -include build/make/test-common.mk -include build/make/test-integration.mk -include build/make/test-unit.mk -include build/make/mocks.mk -include build/make/static-analysis.mk include build/make/clean.mk -# either package-tar.mk -include build/make/package-tar.mk -# or package-debian.mk -include build/make/package-debian.mk -# deploy-debian.mk depends on package-debian.mk -include build/make/deploy-debian.mk include build/make/digital-signature.mk -include build/make/yarn.mk -include build/make/bower.mk -# only include this in repositories which support the automatic release process (like dogus or golang apps) include build/make/release.mk -# either k8s-dogu.mk -include build/make/k8s-dogu.mk -# or k8s-controller.mk; only include this in k8s-controller repositories -include build/make/k8s-controller.mk - +include build/make/bats.mk diff --git a/README.md b/README.md index 5127c68..d8c2a03 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,71 @@ This repository holds makefiles for building Cloudogu tools, especially those wr Please note that `make` only accepts `Makefile`s that are **only** indented with tabs. +## Quick Example + +```makefile +# Set these to the desired values +ARTIFACT_ID= +VERSION= + +MAKEFILES_VERSION=8.0.0 + +.DEFAULT_GOAL:=help + +# set PRE_COMPILE to define steps that shall be executed before the go build +# PRE_COMPILE= + +# set GO_ENV_VARS to define go environment variables for the go build +# GO_ENV_VARS = CGO_ENABLED=0 + +# set PRE_UNITTESTS and POST_UNITTESTS to define steps that shall be executed before or after the unit tests +# PRE_UNITTESTS?= +# POST_UNITTESTS?= + +# set PREPARE_PACKAGE to define a target that should be executed before the package build +# PREPARE_PACKAGE= + +# set ADDITIONAL_CLEAN to define a target that should be executed before the clean target, e.g. +# ADDITIONAL_CLEAN=clean_deb +# clean_deb: +# rm -rf ${DEBIAN_BUILD_DIR} + +# APT_REPO controls the target apt repository for deploy-debian.mk +# -> APT_REPO=ces-premium results in a deploy to the premium apt repository +# -> Everything else results in a deploy to the public repositories +APT_REPO?=ces + +include build/make/variables.mk + +# You may want to overwrite existing variables for target actions to fit into your project. + +include build/make/self-update.mk +include build/make/dependencies-gomod.mk +include build/make/build.mk +include build/make/test-common.mk +include build/make/test-integration.mk +include build/make/test-unit.mk +include build/make/mocks.mk +include build/make/static-analysis.mk +include build/make/clean.mk +# either package-tar.mk +include build/make/package-tar.mk +# or package-debian.mk +include build/make/package-debian.mk +# deploy-debian.mk depends on package-debian.mk +include build/make/deploy-debian.mk +include build/make/digital-signature.mk +include build/make/yarn.mk +include build/make/bower.mk +# only include this in repositories which support the automatic release process (like dogus or golang apps) +include build/make/release.mk +# either k8s-dogu.mk +include build/make/k8s-dogu.mk +# or k8s-controller.mk; only include this in k8s-controller repositories +include build/make/k8s-controller.mk +include build/make/bats.mk +``` + ## Overview over make targets Starting with makefiles v5.0.0 `make help` will produce an overview of make popular targets: @@ -284,5 +349,66 @@ This module enables you to use bower via the `bower-install` target. ### release.mk This module holds the `dogu-release` or other binary release related targets for starting automated production releases. +Additionally, to the regular `dogu-release` the module contains a `dogu-cve-release`. This target checks if a simple +build of a dogu eliminates critical CVEs. If this is the case, a release process will be triggered. Only include this module in dogu or Golang repositories that support a dedicated release flow! +### bats.mk + +This module enables you to run BATS shell tests via the `unit-test-shell` target. All you need is a directory with BATS +tests in `${yourProjectDir}/batsTests` (overrideable with the variable `TESTS_DIR`). + +### K8s-related makefiles + +#### k8s.mk + +This module provides generic targets for developing K8s Cloudogu EcoSystem + +- `image-import` - imports the currently available image into the cluster-local registry. +- `docker-dev-tag` - tags a Docker image for local K8s-CES deployment. +- `docker-build` - builds the docker image of the K8s app. +- `k8s-generate` - generates one concatenated resource YAML +- `k8s-apply` - applies all generated K8s resources to the current cluster and namespace +- check single or all of these variables: + - `check-all-vars` + - `check-k8s-namespace-env-var` + - `check-k8s-image-env-var` + - `check-k8s-artifact-id` + - `check-etc-hosts` + - `check-insecure-cluster-registry` + +#### k8s-component.mk + +This module provides targets for developing K8s Cloudogu EcoSystem components (including controllers) +- General helm targets + - `helm-init-chart` - Creates a Chart.yaml-template with zero values + - `helm-generate-chart` - Generates the final helm chart +- Helm developing targets + - `helm-generate` - Generates the final helm chart with dev-urls + - `helm-apply` - Generates and installs the helm chart + - `helm-delete` - Uninstalls the current helm chart + - `helm-reinstall` - Uninstalls the current helm chart and re-installs it + - `helm-chart-import` - Imports the currently available chart into the cluster-local registry +- Release targets + - `helm-package-release` - Generates and packages the helm chart with release urls. + - `helm-generate-release` - Generates the final helm chart with release urls. +- Component-oriented targets + - `component-generate` - Generate the component YAML resource + - `component-apply` - Applies the component yaml resource to the actual defined context. + - `component-reinstall` - Re-installs the component yaml resource from the actual defined context. + - `component-delete` - Deletes the component yaml resource from the actual defined context. + +#### k8s-dogu.mk + +This module provides targets for developing Dogus with a K8s Cloudogu EcoSystem. + +- `build` - Builds a new version of the dogu and deploys it into the K8s-EcoSystem. +- `install-dogu-descriptor` - Installs a configmap with current dogu.json into the cluster. + +#### k8s-controller.mk + +This module provides targets for K8s Cloudogu EcoSystem controllers. + +- `k8s-integration-test` - Run k8s integration tests. +- `controller-release` - Interactively starts the release workflow. +- `build: helm-apply` - Builds a new version of the dogu and deploys it into the K8s-EcoSystem. diff --git a/batsTests/release_cve.bats b/batsTests/release_cve.bats new file mode 100644 index 0000000..a8c8e17 --- /dev/null +++ b/batsTests/release_cve.bats @@ -0,0 +1,348 @@ +#! /bin/bash +# Bind an unbound BATS variables that fail all tests when combined with 'set -o nounset' +export BATS_TEST_START_TIME="0" +export BATSLIB_FILE_PATH_REM="" +export BATSLIB_FILE_PATH_ADD="" + +load '/workspace/target/bats_libs/bats-support/load.bash' +load '/workspace/target/bats_libs/bats-assert/load.bash' +load '/workspace/target/bats_libs/bats-mock/load.bash' +load '/workspace/target/bats_libs/bats-file/load.bash' + +setup() { + export WORKDIR=/workspace + export MAKE_DIR="${WORKDIR}"/build/make + export PATH="${BATS_TMPDIR}:${PATH}" + docker="$(mock_create)" + ln -s "${docker}" "${BATS_TMPDIR}/docker" + jq="$(mock_create)" + ln -s "${jq}" "${BATS_TMPDIR}/jq" + read="$(mock_create)" + ln -s "${read}" "${BATS_TMPDIR}/read" + release_script="$(mock_create)" + ln -s "${release_script}" "${BATS_TMPDIR}/release" +} + +teardown() { + unset MAKE_DIR + unset WORKDIR + rm "${BATS_TMPDIR}/docker" + rm "${BATS_TMPDIR}/jq" + rm "${BATS_TMPDIR}/read" + rm "${BATS_TMPDIR}/release" +} + +@test "source script with bash should return exit code 0" { + run source "${MAKE_DIR}/release_cve.sh" + + assert_success +} + +@test "diffArrays should print values which are in the first but not in the second array" { + source "${MAKE_DIR}/release_cve.sh" + + run diffArrays "CVE-11111 CVE-22222 CVE-33333" "CVE-22222" + + assert_success + assert_line "CVE-11111 CVE-33333" +} + +@test "diffArrays should print nothing on equal arrays" { + source "${MAKE_DIR}/release_cve.sh" + + local result + result=$(diffArrays "CVE-11111 CVE-22222 CVE-33333" "CVE-11111 CVE-22222 CVE-33333") + + assert_equal "${result}" "" +} + +@test "diffArrays should print nothing on empty arrays" { + source "${MAKE_DIR}/release_cve.sh" + + local result + result=$(diffArrays "" "") + + assert_equal "${result}" "" +} + +@test "docker login should call the login sub command with provided globals" { + source "${MAKE_DIR}/release_cve.sh" + + export USERNAME="user" + export PASSWORD="password" + export REGISTRY_URL="registry" + + run dockerLogin + + assert_success + assert_equal "$(mock_get_call_num "${docker}")" "1" + assert_equal "$(mock_get_call_args "${docker}" "1")" "login registry -u user -p password" +} + +@test "docker logout should call the logout sub command with registry globals" { + source "${MAKE_DIR}/release_cve.sh" + + export REGISTRY_URL="registry" + + run dockerLogout + + assert_success + assert_equal "$(mock_get_call_num "${docker}")" "1" + assert_equal "$(mock_get_call_args "${docker}" "1")" "logout registry" +} + +@test "nameFromDogu should return the name from the dogu.json file" { + source "${MAKE_DIR}/release_cve.sh" + export DOGU_JSON_FILE="dogu.json" + + run nameFromDogu + + assert_success + assert_equal "$(mock_get_call_num "${jq}")" "1" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Name ${DOGU_JSON_FILE}" +} + +@test "versionFromDogu should return the name from the dogu.json file" { + source "${MAKE_DIR}/release_cve.sh" + export DOGU_JSON_FILE="dogu.json" + + run versionFromDogu + + assert_success + assert_equal "$(mock_get_call_num "${jq}")" "1" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Version ${DOGU_JSON_FILE}" +} + +@test "imageFromDogu should return the name from the dogu.json file" { + source "${MAKE_DIR}/release_cve.sh" + export DOGU_JSON_FILE="dogu.json" + + run imageFromDogu + + assert_success + assert_equal "$(mock_get_call_num "${jq}")" "1" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Image ${DOGU_JSON_FILE}" +} + +@test "jsonPropertyFromDogu should call jq with the dogu.json file global variable and the provided parameter" { + source "${MAKE_DIR}/release_cve.sh" + export DOGU_JSON_FILE="dogu.json" + + run jsonPropertyFromDogu "property" + + assert_success + assert_equal "$(mock_get_call_num "${jq}")" "1" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r property ${DOGU_JSON_FILE}" +} + +@test "readCredentialsIfUnset should not ask to enter credentials if they are set" { + source "${MAKE_DIR}/release_cve.sh" + export USERNAME="user" + export PASSWORD="password" + + run readCredentialsIfUnset + + assert_success + assert_equal "$(mock_get_call_num "${read}")" "0" +} + +@test "parseTrivyJsonResult should call jq with given severity and result file" { + source "${MAKE_DIR}/release_cve.sh" + export PASSWORD="password" + + run parseTrivyJsonResult "severity" "result.json" + + assert_success + assert_equal "$(mock_get_call_num "${jq}")" "1" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"severity\") | .VulnerabilityID] | join(\" \") result.json" +} + +@test "runMain should not start release process if cve were added" { + source "${MAKE_DIR}/release_cve.sh" + export TRIVY_PATH="${BATS_TMPDIR}/trivy" + export TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + export TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + export TRIVY_DOCKER_CACHE_DIR=/tmp/db + export TRIVY_IMAGE_SCAN_FLAGS="--use this" + + export USERNAME="user" + export PASSWORD="password" + + mock_set_output "${jq}" "jenkins" "1" + mock_set_output "${jq}" "1.0.0" "2" + mock_set_output "${jq}" "jenkins" "3" + mock_set_output "${jq}" "1.0.0" "4" + mock_set_output "${jq}" "CVE-1" "5" + mock_set_output "${jq}" "jenkins" "6" + mock_set_output "${jq}" "1.0.0" "7" + mock_set_output "${jq}" "jenkins" "8" + mock_set_output "${jq}" "1.0.0" "9" + mock_set_output "${jq}" "CVE-1 CVE-2" "10" + + run runMain + + assert_equal "$(mock_get_call_num "${docker}")" "6" + assert_equal "$(mock_get_call_args "${docker}" "1")" "login registry.cloudogu.com -u user -p password" + assert_equal "$(mock_get_call_args "${docker}" "2")" "pull jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "3")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "4")" "build . -t jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "5")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "6")" "logout registry.cloudogu.com" + assert_equal "$(mock_get_call_num "${jq}")" "10" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "2")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "3")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "4")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "5")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_args "${jq}" "6")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "7")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "8")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "9")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "10")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_line "Abort release. Added new vulnerabilities:" + assert_line "CVE-2" + assert_failure "2" +} + +@test "runMain should not start release process if no cve will be fixed" { + source "${MAKE_DIR}/release_cve.sh" + export TRIVY_PATH="${BATS_TMPDIR}/trivy" + export TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + export TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + export TRIVY_DOCKER_CACHE_DIR=/tmp/db + export TRIVY_IMAGE_SCAN_FLAGS="--use this" + + export USERNAME="user" + export PASSWORD="password" + + mock_set_output "${jq}" "jenkins" "1" + mock_set_output "${jq}" "1.0.0" "2" + mock_set_output "${jq}" "jenkins" "3" + mock_set_output "${jq}" "1.0.0" "4" + mock_set_output "${jq}" "CVE-1 CVE-2" "5" + mock_set_output "${jq}" "jenkins" "6" + mock_set_output "${jq}" "1.0.0" "7" + mock_set_output "${jq}" "jenkins" "8" + mock_set_output "${jq}" "1.0.0" "9" + mock_set_output "${jq}" "CVE-1 CVE-2" "10" + + run runMain + + assert_equal "$(mock_get_call_num "${docker}")" "6" + assert_equal "$(mock_get_call_args "${docker}" "1")" "login registry.cloudogu.com -u user -p password" + assert_equal "$(mock_get_call_args "${docker}" "2")" "pull jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "3")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "4")" "build . -t jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "5")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "6")" "logout registry.cloudogu.com" + assert_equal "$(mock_get_call_num "${jq}")" "10" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "2")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "3")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "4")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "5")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_args "${jq}" "6")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "7")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "8")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "9")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "10")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_line "Abort release. Fixed no new vulnerabilities" + assert_failure "3" +} + +@test "runMain should start release process if cves will be fixed without dry run option" { + source "${MAKE_DIR}/release_cve.sh" + export TRIVY_PATH="${BATS_TMPDIR}/trivy" + export TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + export TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + export TRIVY_DOCKER_CACHE_DIR=/tmp/db + export TRIVY_IMAGE_SCAN_FLAGS="--use this" + export RELEASE_SH="${release_script}" + + export USERNAME="user" + export PASSWORD="password" + + mock_set_output "${jq}" "jenkins" "1" + mock_set_output "${jq}" "1.0.0" "2" + mock_set_output "${jq}" "jenkins" "3" + mock_set_output "${jq}" "1.0.0" "4" + mock_set_output "${jq}" "CVE-1 CVE-2" "5" + mock_set_output "${jq}" "jenkins" "6" + mock_set_output "${jq}" "1.0.0" "7" + mock_set_output "${jq}" "jenkins" "8" + mock_set_output "${jq}" "1.0.0" "9" + mock_set_output "${jq}" "CVE-1" "10" + + run runMain + + assert_equal "$(mock_get_call_num "${docker}")" "6" + assert_equal "$(mock_get_call_args "${docker}" "1")" "login registry.cloudogu.com -u user -p password" + assert_equal "$(mock_get_call_args "${docker}" "2")" "pull jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "3")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "4")" "build . -t jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "5")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "6")" "logout registry.cloudogu.com" + assert_equal "$(mock_get_call_num "${jq}")" "10" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "2")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "3")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "4")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "5")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_args "${jq}" "6")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "7")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "8")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "9")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "10")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_num "${release_script}")" "1" + assert_equal "$(mock_get_call_args "${release_script}" "1")" "dogu-cve-release CVE-2 " + assert_success +} + +@test "runMain should start release process if cves will be fixed with dry run option" { + source "${MAKE_DIR}/release_cve.sh" + export TRIVY_PATH="${BATS_TMPDIR}/trivy" + export TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + export TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + export TRIVY_DOCKER_CACHE_DIR=/tmp/db + export TRIVY_IMAGE_SCAN_FLAGS="--use this" + export RELEASE_SH="${release_script}" + export DRY_RUN="true" + + export USERNAME="user" + export PASSWORD="password" + + mock_set_output "${jq}" "jenkins" "1" + mock_set_output "${jq}" "1.0.0" "2" + mock_set_output "${jq}" "jenkins" "3" + mock_set_output "${jq}" "1.0.0" "4" + mock_set_output "${jq}" "CVE-1 CVE-2" "5" + mock_set_output "${jq}" "jenkins" "6" + mock_set_output "${jq}" "1.0.0" "7" + mock_set_output "${jq}" "jenkins" "8" + mock_set_output "${jq}" "1.0.0" "9" + mock_set_output "${jq}" "CVE-1" "10" + + run runMain + + assert_equal "$(mock_get_call_num "${docker}")" "6" + assert_equal "$(mock_get_call_args "${docker}" "1")" "login registry.cloudogu.com -u user -p password" + assert_equal "$(mock_get_call_args "${docker}" "2")" "pull jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "3")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "4")" "build . -t jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "5")" "run -v ${TRIVY_CACHE_DIR}:/tmp/db -v /var/run/docker.sock:/var/run/docker.sock -v ${TRIVY_PATH}:/result aquasec/trivy --cache-dir ${TRIVY_DOCKER_CACHE_DIR} -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS} jenkins:1.0.0" + assert_equal "$(mock_get_call_args "${docker}" "6")" "logout registry.cloudogu.com" + assert_equal "$(mock_get_call_num "${jq}")" "10" + assert_equal "$(mock_get_call_args "${jq}" "1")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "2")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "3")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "4")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "5")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_args "${jq}" "6")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "7")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "8")" "-r .Image dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "9")" "-r .Version dogu.json" + assert_equal "$(mock_get_call_args "${jq}" "10")" "-rc [.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"CRITICAL\") | .VulnerabilityID] | join(\" \") /tmp/trivy/results.json" + assert_equal "$(mock_get_call_num "${release_script}")" "1" + assert_equal "$(mock_get_call_args "${release_script}" "1")" "dogu-cve-release CVE-2 true" + assert_success +} diff --git a/build/make/bats/Dockerfile b/build/make/bats/Dockerfile index f75afe1..428ee05 100644 --- a/build/make/bats/Dockerfile +++ b/build/make/bats/Dockerfile @@ -4,4 +4,4 @@ ARG BATS_TAG FROM ${BATS_BASE_IMAGE}:${BATS_TAG} # Make bash more findable by scripts and tests -RUN apk add make git bash \ No newline at end of file +RUN apk add make git bash diff --git a/build/make/k8s.mk b/build/make/k8s.mk index 5ea6750..3f2ab94 100644 --- a/build/make/k8s.mk +++ b/build/make/k8s.mk @@ -67,7 +67,7 @@ K8S_PRE_GENERATE_TARGETS ?= k8s-create-temporary-resource k8s-generate: ${BINARY_YQ} $(K8S_RESOURCE_TEMP_FOLDER) $(K8S_PRE_GENERATE_TARGETS) ## Generates the final resource yaml. @echo "Applying general transformations..." @sed -i "s/'{{ .Namespace }}'/$(NAMESPACE)/" $(K8S_RESOURCE_TEMP_YAML) - @if [[ ${STAGE} == "development" ]]; then \ + @if [[ "${STAGE}" == "development" ]]; then \ $(BINARY_YQ) -i e "(select(.kind == \"Deployment\").spec.template.spec.containers[]|select(.image == \"*$(ARTIFACT_ID)*\").image)=\"$(IMAGE_DEV)\"" $(K8S_RESOURCE_TEMP_YAML); \ else \ $(BINARY_YQ) -i e "(select(.kind == \"Deployment\").spec.template.spec.containers[]|select(.image == \"*$(ARTIFACT_ID)*\").image)=\"$(IMAGE)\"" $(K8S_RESOURCE_TEMP_YAML); \ @@ -119,5 +119,8 @@ __check_defined = \ $(if $(value $1),, \ $(error Undefined $1$(if $2, ($2)))) +.PHONY: install-yq ## Installs the yq YAML editor. +install-yq: ${BINARY_YQ} + ${BINARY_YQ}: $(UTILITY_BIN_PATH) ## Download yq locally if necessary. $(call go-get-tool,$(BINARY_YQ),github.com/mikefarah/yq/v4@v4.25.1) diff --git a/build/make/release.mk b/build/make/release.mk index 11dde9a..82ba1ba 100644 --- a/build/make/release.mk +++ b/build/make/release.mk @@ -8,4 +8,8 @@ dogu-release: ## Start a dogu release .PHONY: go-release go-release: ## Start a go tool release - build/make/release.sh go-tool \ No newline at end of file + build/make/release.sh go-tool + +.PHONY: dogu-cve-release +dogu-cve-release: ## Start a dogu release of a new build if the local build fixes critical CVEs + @bash -c "build/make/release_cve.sh \"${REGISTRY_USERNAME}\" \"${REGISTRY_PASSWORD}\" \"${TRIVY_IMAGE_SCAN_FLAGS}\" \"${DRY_RUN}\"" diff --git a/build/make/release.sh b/build/make/release.sh index 4fd4569..ae9a722 100755 --- a/build/make/release.sh +++ b/build/make/release.sh @@ -15,7 +15,7 @@ sourceCustomReleaseArgs() { if [[ -f "${RELEASE_ARGS_FILE}" ]]; then echo "Using custom release args file ${RELEASE_ARGS_FILE}" - sourceCustomReleaseExitCode=0 + local sourceCustomReleaseExitCode=0 # shellcheck disable=SC1090 source "${RELEASE_ARGS_FILE}" || sourceCustomReleaseExitCode=$? if [[ ${sourceCustomReleaseExitCode} -ne 0 ]]; then @@ -30,13 +30,16 @@ RELEASE_ARGS_FILE="${PROJECT_DIR}/release_args.sh" sourceCustomReleaseArgs "${RELEASE_ARGS_FILE}" +# shellcheck disable=SC1090 source "$(pwd)/build/make/release_functions.sh" TYPE="${1}" +FIXED_CVE_LIST="${2:-""}" +DRY_RUN="${3:-""}" echo "=====Starting Release process=====" -if [ "${TYPE}" == "dogu" ];then +if [[ "${TYPE}" == "dogu" || "${TYPE}" == "dogu-cve-release" ]];then CURRENT_TOOL_VERSION=$(get_current_version_by_dogu_json) else CURRENT_TOOL_VERSION=$(get_current_version_by_makefile) @@ -45,10 +48,20 @@ fi NEW_RELEASE_VERSION="$(read_new_version)" validate_new_version "${NEW_RELEASE_VERSION}" -start_git_flow_release "${NEW_RELEASE_VERSION}" +if [[ -n "${DRY_RUN}" ]]; then + start_dry_run_release "${NEW_RELEASE_VERSION}" +else + start_git_flow_release "${NEW_RELEASE_VERSION}" +fi + update_versions "${NEW_RELEASE_VERSION}" -update_changelog "${NEW_RELEASE_VERSION}" +update_changelog "${NEW_RELEASE_VERSION}" "${FIXED_CVE_LIST}" show_diff -finish_release_and_push "${CURRENT_TOOL_VERSION}" "${NEW_RELEASE_VERSION}" + +if [[ -n "${DRY_RUN}" ]]; then + abort_dry_run_release "${NEW_RELEASE_VERSION}" +else + finish_release_and_push "${CURRENT_TOOL_VERSION}" "${NEW_RELEASE_VERSION}" +fi echo "=====Finished Release process=====" diff --git a/build/make/release_cve.sh b/build/make/release_cve.sh new file mode 100755 index 0000000..21738cb --- /dev/null +++ b/build/make/release_cve.sh @@ -0,0 +1,154 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + +function readCredentialsIfUnset() { + if [ -z "${USERNAME}" ]; then + echo "username is unset" + while [[ -z ${USERNAME} ]]; do + read -r -p "type username for ${REGISTRY_URL}: " USERNAME + done + fi + if [ -z "${PASSWORD}" ]; then + echo "password is unset" + while [[ -z ${PASSWORD} ]]; do + read -r -s -p "type password for ${REGISTRY_URL}: " PASSWORD + done + fi +} + +function diffArrays() { + local cveListX=("$1") + local cveListY=("$2") + local result=() + + local cveX + # Disable the following shellcheck because the arrays are sufficiently whitespace delimited because of the jq parsing result. + # shellcheck disable=SC2128 + for cveX in ${cveListX}; do + local found=0 + local cveY + for cveY in ${cveListY}; do + [[ "${cveY}" == "${cveX}" ]] && { + found=1 + break + } + done + + [[ "${found}" == 0 ]] && result+=("${cveX}") + done + + echo "${result[@]}" +} + +function dockerLogin() { + docker login "${REGISTRY_URL}" -u "${USERNAME}" -p "${PASSWORD}" +} + +function dockerLogout() { + docker logout "${REGISTRY_URL}" +} + +function nameFromDogu() { + jsonPropertyFromDogu ".Name" +} + +function imageFromDogu() { + jsonPropertyFromDogu ".Image" +} + +function versionFromDogu() { + jsonPropertyFromDogu ".Version" +} + +function jsonPropertyFromDogu() { + local property="${1}" + jq -r "${property}" "${DOGU_JSON_FILE}" +} + +function pullRemoteImage() { + docker pull "$(imageFromDogu):$(versionFromDogu)" +} + +function buildLocalImage() { + docker build . -t "$(imageFromDogu):$(versionFromDogu)" +} + +function scanImage() { + docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS:+"${TRIVY_IMAGE_SCAN_FLAGS}"} "$(imageFromDogu):$(versionFromDogu)" +} + +function parseTrivyJsonResult() { + local severity="${1}" + local trivy_result_file="${2}" + + # First select results which have the property "Vulnerabilities". Filter the vulnerability ids with the given severity and afterward put the values in an array. + # This array is used to format the values with join(" ") in a whitespace delimited string list. + jq -rc "[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"${severity}\") | .VulnerabilityID] | join(\" \")" "${trivy_result_file}" +} + +RELEASE_SH="build/make/release.sh" + +REGISTRY_URL="registry.cloudogu.com" +DOGU_JSON_FILE="dogu.json" + +CVE_SEVERITY="CRITICAL" + +TRIVY_PATH= +TRIVY_RESULT_FILE= +TRIVY_CACHE_DIR= +TRIVY_DOCKER_CACHE_DIR=/tmp/db +TRIVY_IMAGE_SCAN_FLAGS= + +USERNAME="" +PASSWORD="" +DRY_RUN= + +function runMain() { + readCredentialsIfUnset + dockerLogin + + mkdir -p "${TRIVY_PATH}" # Cache will not be removed after release. rm requires root because the trivy container only runs with root. + pullRemoteImage + scanImage + local remote_trivy_cve_list + remote_trivy_cve_list=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") + + buildLocalImage + scanImage + local local_trivy_cve_list + local_trivy_cve_list=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") + + dockerLogout + + local cve_in_local_but_not_in_remote + cve_in_local_but_not_in_remote=$(diffArrays "${local_trivy_cve_list}" "${remote_trivy_cve_list}") + if [[ -n "${cve_in_local_but_not_in_remote}" ]]; then + echo "Abort release. Added new vulnerabilities:" + echo "${cve_in_local_but_not_in_remote[@]}" + exit 2 + fi + + local cve_in_remote_but_not_in_local + cve_in_remote_but_not_in_local=$(diffArrays "${remote_trivy_cve_list}" "${local_trivy_cve_list}") + if [[ -z "${cve_in_remote_but_not_in_local}" ]]; then + echo "Abort release. Fixed no new vulnerabilities" + exit 3 + fi + + "${RELEASE_SH}" "dogu-cve-release" "${cve_in_remote_but_not_in_local}" "${DRY_RUN}" +} + +# make the script only runMain when executed, not when sourced from bats tests +if [[ -n "${BASH_VERSION}" && "${BASH_SOURCE[0]}" == "${0}" ]]; then + USERNAME="${1:-""}" + PASSWORD="${2:-""}" + TRIVY_IMAGE_SCAN_FLAGS="${3:-""}" + DRY_RUN="${4:-""}" + + TRIVY_PATH="/tmp/trivy-dogu-cve-release-$(nameFromDogu)" + TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + runMain +fi diff --git a/build/make/release_functions.sh b/build/make/release_functions.sh index 528806b..499c248 100755 --- a/build/make/release_functions.sh +++ b/build/make/release_functions.sh @@ -3,15 +3,15 @@ set -o errexit set -o nounset set -o pipefail -wait_for_ok(){ +wait_for_ok() { printf "\n" - OK=false - while [[ ${OK} != "ok" ]] ; do + local OK="false" + while [[ "${OK}" != "ok" ]]; do read -r -p "${1} (type 'ok'): " OK done } -ask_yes_or_no(){ +ask_yes_or_no() { local ANSWER="" while [ "${ANSWER}" != "y" ] && [ "${ANSWER}" != "n" ]; do @@ -21,49 +21,51 @@ ask_yes_or_no(){ echo "${ANSWER}" } -get_current_version_by_makefile(){ +get_current_version_by_makefile() { grep '^VERSION=[0-9[:alpha:].-]*$' Makefile | sed s/VERSION=//g } -get_current_version_by_dogu_json(){ +get_current_version_by_dogu_json() { jq ".Version" --raw-output dogu.json } -read_new_version(){ +read_new_version() { local NEW_RELEASE_VERSION read -r -p "Current Version is v${CURRENT_TOOL_VERSION}. Please provide the new version: v" NEW_RELEASE_VERSION echo "${NEW_RELEASE_VERSION}" } -validate_new_version(){ +validate_new_version() { local NEW_RELEASE_VERSION="${1}" # Validate that release version does not start with vv if [[ ${NEW_RELEASE_VERSION} = v* ]]; then echo "WARNING: The new release version (v${NEW_RELEASE_VERSION}) starts with 'vv'." echo "You must not enter the v when defining the new version." + local ANSWER ANSWER=$(ask_yes_or_no "Should the first v be removed?") if [ "${ANSWER}" == "y" ]; then NEW_RELEASE_VERSION="${NEW_RELEASE_VERSION:1}" echo "Release version now is: ${NEW_RELEASE_VERSION}" fi - fi; + fi } -start_git_flow_release(){ +start_git_flow_release() { local NEW_RELEASE_VERSION="${1}" # Do gitflow git flow init --defaults --force + local mainBranchExists mainBranchExists="$(git show-ref refs/remotes/origin/main || echo "")" if [ -n "$mainBranchExists" ]; then - echo 'Using "main" branch for production releases' - git flow config set master main - git checkout main - git pull origin main + echo 'Using "main" branch for production releases' + git flow config set master main + git checkout main + git pull origin main else - echo 'Using "master" branch for production releases' - git checkout master - git pull origin master + echo 'Using "master" branch for production releases' + git checkout master + git pull origin master fi git checkout develop @@ -71,17 +73,30 @@ start_git_flow_release(){ git flow release start v"${NEW_RELEASE_VERSION}" } +start_dry_run_release() { + local NEW_RELEASE_VERSION="${1}" + + git checkout -b dryrun/v"${NEW_RELEASE_VERSION}" +} + +abort_dry_run_release() { + local NEW_RELEASE_VERSION="${1}" + + git checkout develop + git branch -D dryrun/v"${NEW_RELEASE_VERSION}" +} + # update_versions updates files with the new release version and interactively asks the user for verification. If okay # the updated files will be staged to git and finally committed. # # extension points: # - update_versions_modify_files - update a file with the new version number # - update_versions_stage_modified_files - stage a modified file to prepare the file for the up-coming commit -update_versions(){ +update_versions() { local NEW_RELEASE_VERSION="${1}" if [[ $(type -t update_versions_modify_files) == function ]]; then - preSkriptExitCode=0 + local preSkriptExitCode=0 update_versions_modify_files "${NEW_RELEASE_VERSION}" || preSkriptExitCode=$? if [[ ${preSkriptExitCode} -ne 0 ]]; then echo "ERROR: custom update_versions_modify_files() exited with exit code ${preSkriptExitCode}" @@ -92,7 +107,7 @@ update_versions(){ # Update version in dogu.json if [ -f "dogu.json" ]; then echo "Updating version in dogu.json..." - jq ".Version = \"${NEW_RELEASE_VERSION}\"" dogu.json > dogu2.json && mv dogu2.json dogu.json + jq ".Version = \"${NEW_RELEASE_VERSION}\"" dogu.json >dogu2.json && mv dogu2.json dogu.json fi # Update version in Dockerfile @@ -110,7 +125,7 @@ update_versions(){ # Update version in package.json if [ -f "package.json" ]; then echo "Updating version in package.json..." - jq ".version = \"${NEW_RELEASE_VERSION}\"" package.json > package2.json && mv package2.json package.json + jq ".version = \"${NEW_RELEASE_VERSION}\"" package.json >package2.json && mv package2.json package.json fi # Update version in pom.xml @@ -133,7 +148,7 @@ update_versions(){ fi if [ -f "dogu.json" ]; then - git add dogu.json + git add dogu.json fi if [ -f "Dockerfile" ]; then @@ -155,12 +170,14 @@ update_versions(){ git commit -m "Bump version" } -update_changelog(){ +update_changelog() { local NEW_RELEASE_VERSION="${1}" + local FIXED_CVE_LIST="${2}" # Changelog update + local CURRENT_DATE CURRENT_DATE=$(date --rfc-3339=date) - NEW_CHANGELOG_TITLE="## [v${NEW_RELEASE_VERSION}] - ${CURRENT_DATE}" + local NEW_CHANGELOG_TITLE="## [v${NEW_RELEASE_VERSION}] - ${CURRENT_DATE}" # Check if "Unreleased" tag exists while ! grep --silent "## \[Unreleased\]" CHANGELOG.md; do echo "" @@ -169,6 +186,10 @@ update_changelog(){ wait_for_ok "Please insert a \"## [Unreleased]\" line into CHANGELOG.md now." done + if [[ -n "${FIXED_CVE_LIST}" ]]; then + addFixedCVEListFromReRelease "${FIXED_CVE_LIST}" + fi + # Add new title line to changelog sed -i "s|## \[Unreleased\]|## \[Unreleased\]\n\n${NEW_CHANGELOG_TITLE}|g" CHANGELOG.md @@ -186,8 +207,36 @@ update_changelog(){ git commit -m "Update changelog" } -show_diff(){ - if ! git diff --exit-code > /dev/null; then +# addFixedCVEListFromReRelease is used in dogu cve releases. The method adds the fixed CVEs under the ### Fixed header +# in the unreleased section. +addFixedCVEListFromReRelease() { + local fixed_cve_list="${1}" + + local cve_sed_search="" + local cve_sed_replace="" + local fixed_exists_in_unreleased + fixed_exists_in_unreleased=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^### Fixed$" || true) + if [[ -n "${fixed_exists_in_unreleased}" ]]; then + # extend fixed header with CVEs. + cve_sed_search="^\#\#\# Fixed$" + cve_sed_replace="\#\#\# Fixed\n- Fixed ${fixed_cve_list}" + else + # extend unreleased header with fixed header and CVEs. + cve_sed_search="^\#\# \[Unreleased\]$" + cve_sed_replace="\#\# \[Unreleased\]\n\#\#\# Fixed\n- Fixed ${fixed_cve_list}" + + local any_exists_unreleased + any_exists_unreleased=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^\#\#\# Added$" -e "^\#\#\# Fixed$" -e "^\#\#\# Changed$" || true) + if [[ -n ${any_exists_unreleased} ]]; then + cve_sed_replace+="\n" + fi + fi + + sed -i "0,/${cve_sed_search}/s//${cve_sed_replace}/" CHANGELOG.md +} + +show_diff() { + if ! git diff --exit-code >/dev/null; then echo "There are still uncommitted changes:" echo "" echo "# # # # # # # # # #" @@ -206,7 +255,7 @@ show_diff(){ echo "# # # # # # # # # #" } -finish_release_and_push(){ +finish_release_and_push() { local CURRENT_VERSION="${1}" local NEW_RELEASE_VERSION="${2}"