From 7cae7d8a12542ecd6a2585b6d1ca952f3aed745e Mon Sep 17 00:00:00 2001 From: meiserloh Date: Mon, 8 Apr 2024 13:31:23 +0200 Subject: [PATCH 1/6] #17 update makefiles --- Makefile | 2 +- build/make/coder-lib.sh | 182 ++++++++++++++++++++++++++ build/make/coder.mk | 159 ++++++++++++++++++++++ build/make/k8s-component.mk | 150 +++++++++++++++++++++ build/make/k8s-component.tpl | 13 ++ build/make/k8s-controller.mk | 60 ++------- build/make/k8s-crd.mk | 115 ++++++++++++++++ build/make/k8s-dogu.mk | 41 +++--- build/make/k8s-dogu.tpl | 2 +- build/make/k8s.mk | 95 +++++++++----- build/make/mocks.mk | 4 +- build/make/release.mk | 10 +- build/make/release.sh | 23 +++- build/make/release_cve.sh | 166 +++++++++++++++++++++++ build/make/release_functions.sh | 103 +++++++++++---- build/make/yarn.mk | 30 +++-- installation-scripts/install-mysql.sh | 1 + 17 files changed, 1009 insertions(+), 147 deletions(-) create mode 100755 build/make/coder-lib.sh create mode 100644 build/make/coder.mk create mode 100644 build/make/k8s-component.mk create mode 100644 build/make/k8s-component.tpl create mode 100644 build/make/k8s-crd.mk create mode 100755 build/make/release_cve.sh diff --git a/Makefile b/Makefile index 3931ac9..0f89d20 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -MAKEFILES_VERSION=7.6.0 +MAKEFILES_VERSION=9.0.3 .DEFAULT_GOAL:=dogu-release diff --git a/build/make/coder-lib.sh b/build/make/coder-lib.sh new file mode 100755 index 0000000..2b5d198 --- /dev/null +++ b/build/make/coder-lib.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# a collection of helpful functions to update coder workspaces for rapid development +set -e -u -x -o pipefail + +function getContainerBin() { + if [ -x "$(command -v podman)" ]; then + echo "podman"; + else + echo "docker"; + fi +} + +function getCoderUser() { + # check if coder is installed, so that there is no problem with build and release targets if this is called before + if [ -x "$(command -v coder)" ]; then + coder users show me -o json | jq -r '.username'; + fi +} + +function getAllWorkspaces() { + coder list -c workspace | tail -n+2 +} + +function doesWorkspaceExist() { + coderUser="$1" + workspaceName="$2" + + workspace=$(coder list -a -o json | jq -r "select(.[].owner_name == \"${coderUser}\" and .[].name == \"${workspaceName}\") | .[0].name") + if [ -z "$workspace" ]; then + return 1 #workspace does not exist + else + return 0 + fi +} + +function generateUniqueWorkspaceName() { + local wantedWorkspacePrefix="$1" + # use time to make name unique + local time + time=$(date +'%H-%M-%S') + local lengthOfTime=${#time} + local delimiter='-' + local lengthOfDelimiter=${#delimiter} + # trim prefix, as workspace names are limited to 32 chars + local trimmedPrefix="${wantedWorkspacePrefix:0:$((32 - lengthOfDelimiter - lengthOfTime))}" + local uniqueName="${trimmedPrefix}${delimiter}${time}" + # '--' is forbidden in coder, replace multiple '-' with a single one. + echo "${uniqueName}" | awk '{gsub(/[-]+/,"-")}1' + # returns sth like 'myPrefix-12-45-23' +} + +function buildImage() { + local tag="$1" + local containerBuildDir="${2:-./container}" + local secretDir="${3:-./secrets}" + local containerExec="${4:-podman}" + + # include build-secrets if there are any + local secretArgs=() + if [ -d "$secretDir" ]; then + # shellcheck disable=SC2231 + for secretPath in $secretDir/*; do + # do not match .sh scripts + [[ $secretPath == *.sh ]] && continue + local secretName + secretName=$(basename "$secretPath") + secretArgs+=("--secret=id=$secretName,src=$secretDir/$secretName") + done + fi + + if [ "$containerExec" = "podman" ]; then + $containerExec build -t "$tag" --pull=newer "$containerBuildDir" "${secretArgs[@]}" + else + $containerExec build -t "$tag" --pull "$containerBuildDir" "${secretArgs[@]}" + fi +} + +function doTrivyConvert() { + local trivyFlags=$1 + local outputFile=$2 + local containerExec=$3 + local jsonScanToConvert=$4 + + local containerJsonScanFile="/tmp/scan.json" + + # shellcheck disable=SC2086 + # as globbing is what we want here + "$containerExec" run --rm --pull=always \ + -v trivy-cache:/root/.cache \ + -v "$jsonScanToConvert:$containerJsonScanFile" \ + aquasec/trivy -q \ + convert $trivyFlags "$containerJsonScanFile" > "$outputFile" +} + +function uploadTemplate() { + local templateDir="${1:?"Error. you need to add the template directory as the first parameter"}" + local templateName="${2:?"Error. you need to add the template name as the second parameter"}" + # for terraform variables (not editable by workspace users) + local variablesFile="${templateDir}/variables.yaml" + if [ -f "$variablesFile" ]; then + local doesVariablesFileExist=1 + fi + if ! coder template push -y -d "$templateDir" ${doesVariablesFileExist:+--variables-file "$variablesFile"} "$templateName"; then + # if template does not exist yet, create it in coder + coder template create -y -d "$templateDir" ${doesVariablesFileExist:+--variables-file "$variablesFile"} "$templateName" + fi +} + +function createNewWorkspace() { + local templateName="$1" + local workspaceName="$2" + # 3. param is optional, set it to autofill prompts for coder params + local templateDir="${3-unset}" + local richParametersFile="${templateDir}/rich-parameters.yaml" + if [ -n "${templateDir+x}" ] && [ -f "$richParametersFile" ]; then + local doesRichParametersFileExist=1 + fi + coder create -t "$templateName" -y "$workspaceName" ${doesRichParametersFileExist:+--rich-parameter-file "$richParametersFile"} +} + +function removeAllOtherWorkspaces() { + local CODER_USER="$1" + local WORKSPACE_PREFIX="$2" + local IGNORED_WORKSPACE="$3" + WORKSPACES="$(getAllWorkspaces)" + for ws in $WORKSPACES; do + if [ "$ws" != "$CODER_USER/$IGNORED_WORKSPACE" ] && [[ "$ws" =~ ^"$CODER_USER/$WORKSPACE_PREFIX" ]]; then + echo "delete $ws" + if ! coder delete "$ws" -y; then + #do it twice as podman always throws an error at the first time + coder delete "$ws" -y + fi + fi + done +} + +function updateWorkspace() { + local coderUser="$1" + local workspaceName="$2" + local qualifiedWorkspaceName="$coderUser/$workspaceName" + if ! coder stop "$qualifiedWorkspaceName" -y; then + #do it twice as podman always throws an error at the first time + coder stop "$qualifiedWorkspaceName" -y + fi + coder update "$qualifiedWorkspaceName" +} + +function startTestWorkspace() { + local coderUser="$1" + local templateDir="$2" + local workspacePrefix="$3" + local templateName="$4" + local reuseTestWorkspace="$5" + + local newWorkspaceName + if [ "$reuseTestWorkspace" = false ]; then + newWorkspaceName="$(generateUniqueWorkspaceName "$workspacePrefix")" + # do that before deleting others, so that i don't need to wait + createNewWorkspace "$templateName" "$newWorkspaceName" "$templateDir" + # trim prefix as the name of the workspace can also get trimmed + removeAllOtherWorkspaces "$coderUser" "${workspacePrefix:0:22}" "$newWorkspaceName" + else + newWorkspaceName="$workspacePrefix" + if ! doesWorkspaceExist "$coderUser" "$newWorkspaceName"; then + createNewWorkspace "$templateName" "$newWorkspaceName" "$templateDir" + else + updateWorkspace "$coderUser" "$newWorkspaceName" + fi + fi +} + +function uploadToNexus() { + local fileToUpload="$1" + local fileNameNexus="${fileToUpload##*/}" + local templateName="$2" + local releaseVersion="$3" + local nexusUrl="${4:-https://ecosystem.cloudogu.com/nexus/repository/itz-bund/coder}" + set +x #disable command printing because of the password + curl --progress-bar -u "$(cat secrets/nexus-user):$(cat secrets/nexus-pw)" --upload-file "$fileToUpload" \ + "$nexusUrl/$templateName/$releaseVersion/$fileNameNexus" + set -x +} \ No newline at end of file diff --git a/build/make/coder.mk b/build/make/coder.mk new file mode 100644 index 0000000..07f4d43 --- /dev/null +++ b/build/make/coder.mk @@ -0,0 +1,159 @@ +SHELL := /bin/bash + +IMAGE_TAG?=${IMAGE_REGISTRY}/coder/coder-${TEMPLATE_NAME}:${VERSION} +REUSE_TEST_WORKSPACE?=false + +#BUILD_DIR given via variables.mk +TEMPLATE_DIR=${WORKDIR}/template +CONTAINER_BUILD_DIR=${WORKDIR}/container +SECRETS_DIR=${WORKDIR}/secrets +CODER_LIB_PATH=${BUILD_DIR}/make/coder-lib.sh + +RELEASE_DIR=${WORKDIR}/release +MAKE_CHANGE_TOKEN_DIR=${RELEASE_DIR}/make +CONTAINER_FILE?=${CONTAINER_BUILD_DIR}/Dockerfile +CONTAINER_IMAGE_CHANGE_TOKEN?=${MAKE_CHANGE_TOKEN_DIR}/${TEMPLATE_NAME}_image_id.txt +CONTAINER_IMAGE_TAR?=${RELEASE_DIR}/${TEMPLATE_NAME}.tar +CONTAINER_IMAGE_TARGZ?=${RELEASE_DIR}/${TEMPLATE_NAME}.tar.gz +CONTAINER_IMAGE_TRIVY_SCAN_JSON?=${RELEASE_DIR}/trivy.json +CONTAINER_IMAGE_TRIVY_SCAN_TABLE?=${RELEASE_DIR}/trivy.txt +CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE?=${RELEASE_DIR}/trivy_critical.txt +CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON?=${RELEASE_DIR}/trivy_critical.json + +IMAGE_REGISTRY?=registry.cloudogu.com +IMAGE_REGISTRY_USER_FILE?=${SECRETS_DIR}/harbor-user +IMAGE_REGISTRY_PW_FILE?=${SECRETS_DIR}/harbor-pw + +CHANGELOG_FILE=${WORKDIR}/CHANGELOG.md +TEMPLATE_RELEASE_TAR_GZ=${RELEASE_DIR}/${TEMPLATE_NAME}-template.tar.gz + +TEST_WORKSPACE_PREFIX?=test-${TEMPLATE_NAME} +CODER_USER?=$(shell . ${CODER_LIB_PATH} && getCoderUser) + +CONTAINER_BIN?=$(shell . ${CODER_LIB_PATH} && getContainerBin) +GOPASS_BIN?=$(shell command -v gopass 2> /dev/null) + +EXCLUDED_TEMPLATE_FILES?=rich-parameters.yaml variables.yaml + + +##@ Coder template development + +${SECRETS_DIR}: + mkdir -p ${SECRETS_DIR} + +${IMAGE_REGISTRY_USER_FILE}: ${SECRETS_DIR} +ifeq ($(ENVIRONMENT), local) + @echo "Found developer environment. creating secret ${IMAGE_REGISTRY_USER_FILE}" + @${GOPASS_BIN} show ces/websites/registry.cloudogu.com/robot_coder_jenkins | tail -n 1 | sed -e "s/^username: //" > ${IMAGE_REGISTRY_USER_FILE}; +else + @echo "Found CI environment. Please create secrets yourself" +endif + +${IMAGE_REGISTRY_PW_FILE}: ${SECRETS_DIR} +ifeq ($(ENVIRONMENT), local) + @echo "Found developer environment. creating secret ${IMAGE_REGISTRY_PW_FILE}" + @${GOPASS_BIN} show ces/websites/registry.cloudogu.com/robot_coder_jenkins | head -n 1 > ${IMAGE_REGISTRY_PW_FILE}; +else + @echo "Found CI environment. Please create secrets yourself" +endif + +.PHONY: loadGopassSecrets +loadGopassSecrets: ${IMAGE_REGISTRY_USER_FILE} ${IMAGE_REGISTRY_PW_FILE} ${ADDITIONAL_SECRETS_TARGET} ## load secrets from gopass into secret files, so that the build process works locally + +.PHONY: imageRegistryLogin +imageRegistryLogin: loadGopassSecrets ${IMAGE_REGISTRY_USER_FILE} ${IMAGE_REGISTRY_PW_FILE} ## log in to the registry + @${CONTAINER_BIN} login -u "$$(cat ${IMAGE_REGISTRY_USER_FILE})" --password-stdin '${IMAGE_REGISTRY}' < ${IMAGE_REGISTRY_PW_FILE} + +.PHONY: imageRegistryLogout +imageRegistryLogout: ## log out of the registry + @${CONTAINER_BIN} logout '${IMAGE_REGISTRY}' + +.PHONY: buildImage +buildImage: buildImage-$(ENVIRONMENT) ## build the container image + +.PHONY: buildImage-local +buildImage-local: imageRegistryLogin ${CONTAINER_IMAGE_CHANGE_TOKEN} ## build the container image locally + @echo "if the build is not triggered without a change in the dockerfile, try to delete ${CONTAINER_IMAGE_CHANGE_TOKEN}" + +.PHONY: buildImage-ci +buildImage-ci: ${CONTAINER_IMAGE_CHANGE_TOKEN} ## build the container image without automatic secret management + +${CONTAINER_IMAGE_CHANGE_TOKEN}: ${CONTAINER_FILE} + @. ${CODER_LIB_PATH} && buildImage ${IMAGE_TAG} ${CONTAINER_BUILD_DIR} ${SECRETS_DIR} ${CONTAINER_BIN} + @mkdir -p ${MAKE_CHANGE_TOKEN_DIR} + @${CONTAINER_BIN} image ls --format="{{.ID}}" ${IMAGE_TAG} > ${CONTAINER_IMAGE_CHANGE_TOKEN} + +.PHONY: uploadTemplate +uploadTemplate: ## upload template to coder server + @. ${CODER_LIB_PATH} && uploadTemplate ${TEMPLATE_DIR} ${TEMPLATE_NAME} + +.PHONY: startTestWorkspace +startTestWorkspace: ## start a test workspace with coder + @. ${CODER_LIB_PATH} && startTestWorkspace ${CODER_USER} ${TEMPLATE_DIR} ${TEST_WORKSPACE_PREFIX} ${TEMPLATE_NAME} ${REUSE_TEST_WORKSPACE} + +.PHONY: createImageRelease +createImageRelease: ${CONTAINER_IMAGE_TARGZ} ## export the container image as a tar.gz + +${CONTAINER_IMAGE_TAR}: ${CONTAINER_IMAGE_CHANGE_TOKEN} + ${CONTAINER_BIN} save "${IMAGE_TAG}" -o ${CONTAINER_IMAGE_TAR} + +${CONTAINER_IMAGE_TARGZ}: ${CONTAINER_IMAGE_TAR} + gzip -f --keep "${CONTAINER_IMAGE_TAR}" + +.PHONY: trivyscanImage +trivyscanImage: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ## do a trivy scan for the workspace image in various output formats + +${CONTAINER_IMAGE_TRIVY_SCAN_JSON}: ${CONTAINER_IMAGE_TAR} + ${CONTAINER_BIN} run --rm --pull=always \ + -v "trivy-cache:/root/.cache" \ + -v "${CONTAINER_IMAGE_TAR}:/tmp/image.tar" \ + aquasec/trivy -q \ + image --scanners vuln --input /tmp/image.tar -f json --timeout 15m \ + > ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + +${CONTAINER_IMAGE_TRIVY_SCAN_TABLE}: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + @. ${CODER_LIB_PATH} && \ + doTrivyConvert "--format table" ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${CONTAINER_BIN} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + +${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE}: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + @. ${CODER_LIB_PATH} && \ + doTrivyConvert "--format table --severity CRITICAL" ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${CONTAINER_BIN} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + +${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON}: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + @. ${CODER_LIB_PATH} && \ + doTrivyConvert "--format json --severity CRITICAL" ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ${CONTAINER_BIN} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + +.PHONY: createTemplateRelease +createTemplateRelease: ## generate template.tar.gz with all files needed for customers + # remove release dir first as 'cp' cannot merge and will place the source dir inside the target dir if it already exists + rm -rf "${RELEASE_DIR}/${TEMPLATE_NAME}" + cp -r "${TEMPLATE_DIR}" "${RELEASE_DIR}/${TEMPLATE_NAME}/" + #copy changelog + cp "${CHANGELOG_FILE}" "${RELEASE_DIR}/${TEMPLATE_NAME}/" + # remove excludes + for file in "${EXCLUDED_TEMPLATE_FILES}"; do \ + rm -f "${RELEASE_DIR}/${TEMPLATE_NAME}/$$file"; \ + done + tar -czf "${RELEASE_DIR}/${TEMPLATE_NAME}-template.tar.gz" -C "${RELEASE_DIR}" "${TEMPLATE_NAME}" + +.PHONY: createRelease ## generate template- and container archives and the trivy scans +createRelease: createTemplateRelease ${CONTAINER_IMAGE_TARGZ} trivyscanImage ## create the image.tar.gz, template.tar.gz and trivy scans + +.PHONY: cleanCoderRelease +cleanCoderRelease: ## clean release directory + rm -rf "${RELEASE_DIR}" + mkdir -p "${RELEASE_DIR}" + +.PHONY: pushImage +pushImage: ## push the container image into the registry + ${CONTAINER_BIN} push ${IMAGE_TAG} + +.PHONY: uploadRelease +uploadRelease: createTemplateRelease ${CONTAINER_IMAGE_TARGZ} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ## upload release artifacts to nexus + @. ${CODER_LIB_PATH} && uploadToNexus ${TEMPLATE_RELEASE_TAR_GZ} ${TEMPLATE_NAME} ${VERSION} + @. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} ${TEMPLATE_NAME} ${VERSION} + @. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${TEMPLATE_NAME} ${VERSION} + @. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${TEMPLATE_NAME} ${VERSION} + @. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ${TEMPLATE_NAME} ${VERSION} + @. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TARGZ} ${TEMPLATE_NAME} ${VERSION} + diff --git a/build/make/k8s-component.mk b/build/make/k8s-component.mk new file mode 100644 index 0000000..9d29183 --- /dev/null +++ b/build/make/k8s-component.mk @@ -0,0 +1,150 @@ +COMPONENT_DEV_VERSION?=${VERSION}-dev + +include ${BUILD_DIR}/make/k8s.mk + +BINARY_HELM_ADDITIONAL_PUSH_ARGS?=--plain-http +BINARY_HELM_ADDITIONAL_PACK_ARGS?= +BINARY_HELM_ADDITIONAL_UNINST_ARGS?= +BINARY_HELM_ADDITIONAL_UPGR_ARGS?= + +HELM_TARGET_DIR ?= $(K8S_RESOURCE_TEMP_FOLDER)/helm +HELM_SOURCE_DIR ?= k8s/helm +HELM_RELEASE_TGZ=${HELM_TARGET_DIR}/${ARTIFACT_ID}-${VERSION}.tgz +HELM_DEV_RELEASE_TGZ=${HELM_TARGET_DIR}/${ARTIFACT_ID}-${COMPONENT_DEV_VERSION}.tgz +HELM_ARTIFACT_NAMESPACE?=k8s + +K8S_RESOURCE_COMPONENT ?= "${K8S_RESOURCE_TEMP_FOLDER}/component-${ARTIFACT_ID}-${VERSION}.yaml" +K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML ?= $(BUILD_DIR)/make/k8s-component.tpl +# HELM_PRE_GENERATE_TARGETS allows to execute targets that affect Helm source files AND Helm target files. +HELM_PRE_GENERATE_TARGETS ?= +# HELM_POST_GENERATE_TARGETS allows to execute targets that only affect Helm target files. +HELM_POST_GENERATE_TARGETS ?= +HELM_PRE_APPLY_TARGETS ?= +COMPONENT_PRE_APPLY_TARGETS ?= + +# This can be used by components with own images to build and push to the dev registry. +# These components should override this variable with `image-import`. +IMAGE_IMPORT_TARGET?= + +##@ K8s - Helm general +.PHONY: helm-init-chart +helm-init-chart: ${BINARY_HELM} ## Creates a Chart.yaml-template with zero values + @echo "Initialize ${HELM_SOURCE_DIR}/Chart.yaml..." + @mkdir -p ${HELM_SOURCE_DIR}/tmp/ + @${BINARY_HELM} create ${HELM_SOURCE_DIR}/tmp/${ARTIFACT_ID} + @cp ${HELM_SOURCE_DIR}/tmp/${ARTIFACT_ID}/Chart.yaml ${HELM_SOURCE_DIR}/ + @rm -dr ${HELM_SOURCE_DIR}/tmp + @sed -i 's/appVersion: ".*"/appVersion: "0.0.0-replaceme"/' ${HELM_SOURCE_DIR}/Chart.yaml + @sed -i 's/version: .*/version: 0.0.0-replaceme/' ${HELM_SOURCE_DIR}/Chart.yaml + +.PHONY: helm-generate +helm-generate: ${HELM_TARGET_DIR}/Chart.yaml ${HELM_POST_GENERATE_TARGETS} ## Generates the final helm chart. + +# this is phony because of it is easier this way than the makefile-single-run way +.PHONY: ${HELM_TARGET_DIR}/Chart.yaml +${HELM_TARGET_DIR}/Chart.yaml: $(K8S_RESOURCE_TEMP_FOLDER) validate-chart ${HELM_PRE_GENERATE_TARGETS} copy-helm-files + @echo "Generate Helm chart..." + @if [[ ${STAGE} == "development" ]]; then \ + sed -i 's/appVersion: "0.0.0-replaceme"/appVersion: '$(COMPONENT_DEV_VERSION)'/' ${HELM_TARGET_DIR}/Chart.yaml; \ + sed -i 's/version: 0.0.0-replaceme/version: '$(COMPONENT_DEV_VERSION)'/' ${HELM_TARGET_DIR}/Chart.yaml; \ + else \ + sed -i 's/appVersion: "0.0.0-replaceme"/appVersion: "${VERSION}"/' ${HELM_TARGET_DIR}/Chart.yaml; \ + sed -i 's/version: 0.0.0-replaceme/version: ${VERSION}/' ${HELM_TARGET_DIR}/Chart.yaml; \ + fi + +.PHONY: copy-helm-files +copy-helm-files: + @echo "Copying Helm files..." + @rm -drf ${HELM_TARGET_DIR} # delete folder, so the chart is newly created. + @mkdir -p ${HELM_TARGET_DIR}/templates + @cp -r ${HELM_SOURCE_DIR}/** ${HELM_TARGET_DIR} + +.PHONY: validate-chart +validate-chart: + @if [ ! -f ${HELM_SOURCE_DIR}/Chart.yaml ] ; then \ + echo "Could not find source Helm chart under \$${HELM_SOURCE_DIR}/Chart.yaml" ; \ + exit 22 ; \ + fi + +.PHONY: helm-update-dependencies +helm-update-dependencies: ${BINARY_HELM} ## Update Helm chart dependencies + @$(BINARY_HELM) dependency update "${HELM_SOURCE_DIR}" + +##@ K8s - Helm dev targets + +.PHONY: helm-apply +helm-apply: ${BINARY_HELM} check-k8s-namespace-env-var ${IMAGE_IMPORT_TARGET} helm-generate ${HELM_PRE_APPLY_TARGETS} ## Generates and installs the Helm chart. + @echo "Apply generated helm chart" + @${BINARY_HELM} upgrade -i ${ARTIFACT_ID} ${HELM_TARGET_DIR} ${BINARY_HELM_ADDITIONAL_UPGR_ARGS} --namespace ${NAMESPACE} + +.PHONY: helm-delete +helm-delete: ${BINARY_HELM} check-k8s-namespace-env-var ## Uninstalls the current Helm chart. + @echo "Uninstall helm chart" + @${BINARY_HELM} uninstall ${ARTIFACT_ID} --namespace=${NAMESPACE} ${BINARY_HELM_ADDITIONAL_UNINST_ARGS} || true + +.PHONY: helm-reinstall +helm-reinstall: helm-delete helm-apply ## Uninstalls the current helm chart and reinstalls it. + +.PHONY: helm-chart-import +helm-chart-import: ${CHECK_VAR_TARGETS} helm-generate helm-package ${IMAGE_IMPORT_TARGET} ## Imports the currently available chart into the cluster-local registry. + @if [[ ${STAGE} == "development" ]]; then \ + echo "Import ${HELM_DEV_RELEASE_TGZ} into K8s cluster ${K3CES_REGISTRY_URL_PREFIX}..."; \ + ${BINARY_HELM} push ${HELM_DEV_RELEASE_TGZ} oci://${K3CES_REGISTRY_URL_PREFIX}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + else \ + echo "Import ${HELM_RELEASE_TGZ} into K8s cluster ${K3CES_REGISTRY_URL_PREFIX}..."; \ + ${BINARY_HELM} push ${HELM_RELEASE_TGZ} oci://${K3CES_REGISTRY_URL_PREFIX}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + fi + @echo "Done." + +##@ K8s - Helm release targets + +.PHONY: helm-generate-release +helm-generate-release: update-urls ## Generates the final helm chart with release URLs. + + +.PHONY: helm-package +helm-package: helm-delete-existing-tgz ${HELM_RELEASE_TGZ} ## Generates and packages the helm chart with release URLs. + +${HELM_RELEASE_TGZ}: ${BINARY_HELM} ${HELM_TARGET_DIR}/Chart.yaml ${HELM_POST_GENERATE_TARGETS} ## Generates and packages the helm chart with release URLs. + @echo "Package generated helm chart" + @if [[ ${STAGE} == "development" ]]; then \ + echo "WARNING: You are using a development environment" ; \ + fi + @${BINARY_HELM} package ${HELM_TARGET_DIR} -d ${HELM_TARGET_DIR} ${BINARY_HELM_ADDITIONAL_PACK_ARGS} + +.PHONY: helm-delete-existing-tgz +helm-delete-existing-tgz: ## Remove an existing Helm package from the target directory. + @echo "Delete ${HELM_RELEASE_TGZ}*" + @rm -f ${HELM_RELEASE_TGZ}* + +##@ K8s - Helm lint targets + +.PHONY: helm-lint +helm-lint: $(BINARY_HELM) helm-generate + @$(BINARY_HELM) lint "${HELM_TARGET_DIR}" + +##@ K8s - Component dev targets + +.PHONY: component-generate +component-generate: ${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML} ${COMPONENT_POST_GENERATE_TARGETS} ## Generate the component yaml resource. + +${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML}: ${K8S_RESOURCE_TEMP_FOLDER} + @echo "Generating temporary K8s component resource: ${K8S_RESOURCE_COMPONENT}" + @if [[ ${STAGE} == "development" ]]; then \ + sed "s|NAMESPACE|$(HELM_ARTIFACT_NAMESPACE)|g" "${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML}" | sed "s|NAME|$(ARTIFACT_ID)|g" | sed "s|VERSION|$(COMPONENT_DEV_VERSION)|g" > "${K8S_RESOURCE_COMPONENT}"; \ + else \ + sed "s|NAMESPACE|$(HELM_ARTIFACT_NAMESPACE)|g" "${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML}" | sed "s|NAME|$(ARTIFACT_ID)|g" | sed "s|VERSION|$(VERSION)|g" > "${K8S_RESOURCE_COMPONENT}"; \ + fi + +.PHONY: component-apply +component-apply: check-k8s-namespace-env-var ${COMPONENT_PRE_APPLY_TARGETS} ${IMAGE_IMPORT_TARGET} helm-generate helm-chart-import component-generate ## Applies the component yaml resource to the actual defined context. + @kubectl apply -f "${K8S_RESOURCE_COMPONENT}" --namespace="${NAMESPACE}" + @echo "Done." + +.PHONY: component-delete +component-delete: check-k8s-namespace-env-var component-generate $(K8S_POST_GENERATE_TARGETS) ## Deletes the component yaml resource from the actual defined context. + @kubectl delete -f "${K8S_RESOURCE_COMPONENT}" --namespace="${NAMESPACE}" || true + @echo "Done." + +.PHONY: component-reinstall +component-reinstall: component-delete component-apply ## Reinstalls the component yaml resource from the actual defined context. diff --git a/build/make/k8s-component.tpl b/build/make/k8s-component.tpl new file mode 100644 index 0000000..fa0eaa6 --- /dev/null +++ b/build/make/k8s-component.tpl @@ -0,0 +1,13 @@ +# Use the property .spec.deployNamespace to define the namespace the component should be deployed to. +# Make environment variable 'COMPONENT_DEPLOY_NAMESPACE' is responsible for that. +# If 'COMPONENT_DEPLOY_NAMESPACE' is empty the property 'deployNamespace' will be deleted. +apiVersion: k8s.cloudogu.com/v1 +kind: Component +metadata: + name: NAME + labels: + app: ces +spec: + name: NAME + namespace: NAMESPACE + version: VERSION \ No newline at end of file diff --git a/build/make/k8s-controller.mk b/build/make/k8s-controller.mk index 5735f00..ea3d457 100644 --- a/build/make/k8s-controller.mk +++ b/build/make/k8s-controller.mk @@ -1,29 +1,9 @@ -# This script can be used to build and deploy kubernetes controllers. It is required to implement the controller -# specific targets `manifests` and `generate`: -# -# Examples: -# -#.PHONY: manifests -#manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. -# @echo "Generate manifests..." -# @$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases -# -#.PHONY: generate -#generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. -# @echo "Auto-generate deepcopy functions..." -# @$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - # This script requires the k8s.mk script -include $(WORKDIR)/build/make/k8s.mk +include ${BUILD_DIR}/make/k8s-component.mk +include ${BUILD_DIR}/make/k8s-crd.mk ## Variables -# Setting SHELL to bash allows bash commands to be executed by recipes. -# This is a requirement for 'setup-envtest.sh' in the test target. -# Options are set to exit when a recipe line exits non-zero or a piped command fails. -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec - # make sure to create a statically linked binary otherwise it may quit with # "exec user process caused: no such file or directory" GO_BUILD_FLAGS=-mod=vendor -a -tags netgo,osusergo $(LDFLAGS) -o $(BINARY) @@ -38,7 +18,7 @@ K8S_INTEGRATION_TEST_DIR=${TARGET_DIR}/k8s-integration-test ##@ K8s - EcoSystem .PHONY: build -build: image-import k8s-apply ## Builds a new version of the dogu and deploys it into the K8s-EcoSystem. +build: helm-apply ## Builds a new version of the dogu and deploys it into the K8s-EcoSystem. ##@ Release @@ -55,7 +35,7 @@ build-controller: ${SRC} compile ## Builds the controller Go binary. # Allows to perform tasks before locally running the controller K8S_RUN_PRE_TARGETS ?= .PHONY: run -run: manifests generate $(K8S_RUN_PRE_TARGETS) ## Run a controller from your host. +run: generate-deepcopy $(K8S_RUN_PRE_TARGETS) ## Run a controller from your host. go run -ldflags "-X main.Version=$(VERSION)" ./main.go ##@ K8s - Integration test with envtest @@ -64,33 +44,13 @@ $(K8S_INTEGRATION_TEST_DIR): @mkdir -p $@ .PHONY: k8s-integration-test -k8s-integration-test: $(K8S_INTEGRATION_TEST_DIR) manifests generate envtest ## Run k8s integration tests. +k8s-integration-test: $(K8S_INTEGRATION_TEST_DIR) ${ENVTEST} ## Run k8s integration tests. @echo "Running K8s integration tests..." @KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -tags=k8s_integration ./... -coverprofile ${K8S_INTEGRATION_TEST_DIR}/report-k8s-integration.out -##@ K8s - Controller Resource - -# The pre generation script creates a K8s resource yaml containing generated manager yaml. -.PHONY: k8s-create-temporary-resource - k8s-create-temporary-resource: $(K8S_RESOURCE_TEMP_FOLDER) manifests kustomize - @echo "Generating temporary k8s resources $(K8S_RESOURCE_TEMP_YAML)..." - cd $(WORKDIR)/config/manager && $(KUSTOMIZE) edit set image controller=$(IMAGE) - $(KUSTOMIZE) build config/default > $(K8S_RESOURCE_TEMP_YAML) - @echo "Done." - -##@ K8s - Download Kubernetes Utility Tools - -CONTROLLER_GEN = $(UTILITY_BIN_PATH)/controller-gen -.PHONY: controller-gen -controller-gen: ## Download controller-gen locally if necessary. - $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.3) - -KUSTOMIZE = $(UTILITY_BIN_PATH)/kustomize -.PHONY: kustomize -kustomize: ## Download kustomize locally if necessary. - $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.7) +##@ Controller specific targets -ENVTEST = $(UTILITY_BIN_PATH)/setup-envtest -.PHONY: envtest -envtest: ## Download envtest-setup locally if necessary. - $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) \ No newline at end of file +.PHONY: generate-deepcopy +generate-deepcopy: ${CONTROLLER_GEN} ## Generate code containing DeepCopy* method implementations. + @echo "Auto-generate deepcopy functions..." + @$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." diff --git a/build/make/k8s-crd.mk b/build/make/k8s-crd.mk new file mode 100644 index 0000000..4cbcd88 --- /dev/null +++ b/build/make/k8s-crd.mk @@ -0,0 +1,115 @@ +ARTIFACT_CRD_ID = $(ARTIFACT_ID)-crd +DEV_CRD_VERSION ?= ${VERSION}-dev +HELM_CRD_SOURCE_DIR ?= ${WORKDIR}/k8s/helm-crd +HELM_CRD_TARGET_DIR ?= $(K8S_RESOURCE_TEMP_FOLDER)/helm-crd +HELM_CRD_RELEASE_TGZ = ${HELM_CRD_TARGET_DIR}/${ARTIFACT_CRD_ID}-${VERSION}.tgz +HELM_CRD_DEV_RELEASE_TGZ = ${HELM_CRD_TARGET_DIR}/${ARTIFACT_CRD_ID}-${DEV_CRD_VERSION}.tgz + +K8S_RESOURCE_CRD_COMPONENT ?= "${K8S_RESOURCE_TEMP_FOLDER}/component-${ARTIFACT_CRD_ID}-${VERSION}.yaml" +K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML ?= $(BUILD_DIR)/make/k8s-component.tpl +# CRD_POST_MANIFEST_TARGETS can be used to post-process CRD YAMLs after their creation. +CRD_POST_MANIFEST_TARGETS ?= crd-add-labels + +# This can be used by external components to prevent generate and copy controller manifests by overriding with an empty value. +CRD_HELM_MANIFEST_TARGET?=manifests + +##@ K8s - CRD targets + +.PHONY: manifests +manifests: ${CONTROLLER_GEN} manifests-run ${CRD_POST_MANIFEST_TARGETS} ## Generate CustomResourceDefinition YAMLs. + +.PHONY: manifests-run +manifests-run: + @echo "Generate manifests..." + @$(CONTROLLER_GEN) crd paths="./..." output:crd:artifacts:config=${HELM_CRD_SOURCE_DIR}/templates + +.PHONY: crd-add-labels +crd-add-labels: $(BINARY_YQ) + @echo "Adding labels to CRD..." + @for file in ${HELM_CRD_SOURCE_DIR}/templates/*.yaml ; do \ + $(BINARY_YQ) -i e ".metadata.labels.app = \"ces\"" $${file} ;\ + $(BINARY_YQ) -i e ".metadata.labels.\"app.kubernetes.io/name\" = \"${ARTIFACT_ID}\"" $${file} ;\ + done + +.PHONY: crd-helm-generate ## Generates the Helm CRD chart +crd-helm-generate: ${CRD_HELM_MANIFEST_TARGET} validate-crd-chart ${HELM_CRD_TARGET_DIR}/Chart.yaml ${K8S_POST_CRD_HELM_GENERATE_TARGETS} + +# this is phony because of it is easier this way than the makefile-single-run way +.PHONY: ${HELM_CRD_TARGET_DIR}/Chart.yaml +${HELM_CRD_TARGET_DIR}/Chart.yaml: ${K8S_RESOURCE_TEMP_FOLDER} + @echo "Copying Helm CRD files..." + @rm -drf ${HELM_CRD_TARGET_DIR}/templates + @mkdir -p ${HELM_CRD_TARGET_DIR}/templates + @cp -r ${HELM_CRD_SOURCE_DIR}/** ${HELM_CRD_TARGET_DIR} + + @echo "Generate Helm CRD chart..." + @sed -i 's/name: artifact-crd-replaceme/name: ${ARTIFACT_CRD_ID}/' ${HELM_CRD_TARGET_DIR}/Chart.yaml + @if [[ ${STAGE} == "development" ]]; then \ + sed -i 's/appVersion: "0.0.0-replaceme"/appVersion: "${DEV_CRD_VERSION}"/' ${HELM_CRD_TARGET_DIR}/Chart.yaml; \ + sed -i 's/version: 0.0.0-replaceme/version: ${DEV_CRD_VERSION}/' ${HELM_CRD_TARGET_DIR}/Chart.yaml; \ + else \ + sed -i 's/appVersion: "0.0.0-replaceme"/appVersion: "${VERSION}"/' ${HELM_CRD_TARGET_DIR}/Chart.yaml; \ + sed -i 's/version: 0.0.0-replaceme/version: ${VERSION}/' ${HELM_CRD_TARGET_DIR}/Chart.yaml; \ + fi + +.PHONY: validate-crd-chart +validate-crd-chart: + @if [ ! -f ${HELM_CRD_SOURCE_DIR}/Chart.yaml ] ; then \ + echo "Could not find CRD source Helm chart under \$${HELM_CRD_SOURCE_DIR}/Chart.yaml" ; \ + exit 23 ; \ + fi + +.PHONY: crd-helm-apply +crd-helm-apply: ${BINARY_HELM} check-k8s-namespace-env-var crd-helm-generate ## Generates and installs the Helm CRD chart. + @echo "Apply generated Helm CRD chart" + @${BINARY_HELM} upgrade -i ${ARTIFACT_CRD_ID} ${HELM_CRD_TARGET_DIR} ${BINARY_HELM_ADDITIONAL_UPGR_ARGS} --namespace ${NAMESPACE} + +.PHONY: crd-helm-delete +crd-helm-delete: ${BINARY_HELM} check-k8s-namespace-env-var ## Uninstalls the current Helm CRD chart. + @echo "Uninstall Helm CRD chart" + @${BINARY_HELM} uninstall ${ARTIFACT_CRD_ID} --namespace=${NAMESPACE} ${BINARY_HELM_ADDITIONAL_UNINST_ARGS} || true + +.PHONY: crd-helm-package +crd-helm-package: crd-helm-delete-existing-tgz ${HELM_CRD_RELEASE_TGZ} ## Generates and packages the Helm CRD chart. + +.PHONY: crd-helm-delete-existing-tgz +crd-helm-delete-existing-tgz: ## Remove an existing Helm CRD package. + @rm -f ${HELM_CRD_RELEASE_TGZ}* + +${HELM_CRD_RELEASE_TGZ}: ${BINARY_HELM} crd-helm-generate ## Generates and packages the Helm CRD chart. + @echo "Package generated helm crd-chart" + @${BINARY_HELM} package ${HELM_CRD_TARGET_DIR} -d ${HELM_CRD_TARGET_DIR} ${BINARY_HELM_ADDITIONAL_PACK_ARGS} + +.PHONY: crd-helm-chart-import +crd-helm-chart-import: ${CHECK_VAR_TARGETS} check-k8s-artifact-id crd-helm-generate crd-helm-package ## Imports the currently available Helm CRD chart into the cluster-local registry. + @if [[ ${STAGE} == "development" ]]; then \ + echo "Import ${HELM_CRD_DEV_RELEASE_TGZ} into K8s cluster ${K3CES_REGISTRY_URL_PREFIX}..."; \ + ${BINARY_HELM} push ${HELM_CRD_DEV_RELEASE_TGZ} oci://${K3CES_REGISTRY_URL_PREFIX}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + else \ + echo "Import ${HELM_CRD_RELEASE_TGZ} into K8s cluster ${K3CES_REGISTRY_URL_PREFIX}..."; \ + ${BINARY_HELM} push ${HELM_CRD_RELEASE_TGZ} oci://${K3CES_REGISTRY_URL_PREFIX}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + fi + @echo "Done." + +.PHONY: crd-helm-lint +crd-helm-lint: $(BINARY_HELM) crd-helm-generate + @$(BINARY_HELM) lint "${HELM_CRD_TARGET_DIR}" + +.PHONY: crd-component-generate +crd-component-generate: ${K8S_RESOURCE_TEMP_FOLDER} ## Generate the CRD component YAML resource. + @echo "Generating temporary K8s crd-component resource: ${K8S_RESOURCE_CRD_COMPONENT}" + @if [[ ${STAGE} == "development" ]]; then \ + sed "s|NAMESPACE|$(HELM_ARTIFACT_NAMESPACE)|g" "${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML}" | sed "s|NAME|$(ARTIFACT_CRD_ID)|g" | sed "s|VERSION|$(DEV_CRD_VERSION)|g" > "${K8S_RESOURCE_CRD_COMPONENT}"; \ + else \ + sed "s|NAMESPACE|$(HELM_ARTIFACT_NAMESPACE)|g" "${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML}" | sed "s|NAME|$(ARTIFACT_CRD_ID)|g" | sed "s|VERSION|$(VERSION)|g" > "${K8S_RESOURCE_CRD_COMPONENT}"; \ + fi + +.PHONY: crd-component-apply +crd-component-apply: check-k8s-namespace-env-var crd-helm-chart-import crd-component-generate ## Applies the CRD component YAML resource to the actual defined context. + @kubectl apply -f "${K8S_RESOURCE_CRD_COMPONENT}" --namespace="${NAMESPACE}" + @echo "Done." + +.PHONY: crd-component-delete +crd-component-delete: check-k8s-namespace-env-var crd-component-generate ## Deletes the CRD component YAML resource from the actual defined context. + @kubectl delete -f "${K8S_RESOURCE_CRD_COMPONENT}" --namespace="${NAMESPACE}" || true + @echo "Done." diff --git a/build/make/k8s-dogu.mk b/build/make/k8s-dogu.mk index ff0973b..296b1c7 100644 --- a/build/make/k8s-dogu.mk +++ b/build/make/k8s-dogu.mk @@ -1,44 +1,49 @@ # Variables # Path to the dogu json of the dogu -DOGU_JSON_FILE=$(WORKDIR)/dogu.json -DOGU_JSON_DEV_FILE=${TARGET_DIR}/dogu.json +DOGU_JSON_FILE=${WORKDIR}/dogu.json +DOGU_JSON_DEV_FILE=${WORKDIR}/${TARGET_DIR}/dogu.json # Name of the dogu is extracted from the dogu.json -ARTIFACT_ID=$(shell $(BINARY_YQ) -e ".Name" $(DOGU_JSON_FILE) | sed "s|.*/||g") +ARTIFACT_ID=$(shell $(BINARY_YQ) -oy -e ".Name" $(DOGU_JSON_FILE) | sed "s|.*/||g") # Namespace of the dogu is extracted from the dogu.json -ARTIFACT_NAMESPACE=$(shell $(BINARY_YQ) -e ".Name" $(DOGU_JSON_FILE) | sed "s|/.*||g") -# Namespace of the dogu is extracted from the dogu.json -VERSION=$(shell $(BINARY_YQ) -e ".Version" $(DOGU_JSON_FILE)) +ARTIFACT_NAMESPACE=$(shell $(BINARY_YQ) -oy -e ".Name" $(DOGU_JSON_FILE) | sed "s|/.*||g") +# Version of the dogu is extracted from the dogu.json +VERSION=$(shell $(BINARY_YQ) -oy -e ".Version" $(DOGU_JSON_FILE)) # Image of the dogu is extracted from the dogu.json -IMAGE=$(shell $(BINARY_YQ) -e ".Image" $(DOGU_JSON_FILE)):$(VERSION) -IMAGE_DEV_WITHOUT_TAG=$(shell $(BINARY_YQ) -e ".Image" $(DOGU_JSON_FILE) | sed "s|registry\.cloudogu\.com\(.\+\)|${K3CES_REGISTRY_URL_PREFIX}\1|g") -IMAGE_DEV=${IMAGE_DEV_WITHOUT_TAG}:${VERSION} +IMAGE=$(shell $(BINARY_YQ) -oy -e ".Image" $(DOGU_JSON_FILE)):$(VERSION) +IMAGE_DEV_WITHOUT_TAG=$(shell $(BINARY_YQ) -oy -e ".Image" $(DOGU_JSON_FILE) | sed "s|registry\.cloudogu\.com\(.\+\)|${K3CES_REGISTRY_URL_PREFIX}\1|g") +IMAGE_DEV=${IMAGE_DEV_WITHOUT_TAG} -include $(WORKDIR)/build/make/k8s.mk +include $(BUILD_DIR)/make/k8s.mk ##@ K8s - EcoSystem .PHONY: build -build: image-import install-dogu-descriptor k8s-apply ## Builds a new version of the dogu and deploys it into the K8s-EcoSystem. +build: image-import install-dogu-descriptor create-dogu-resource apply-dogu-resource ## Builds a new version of the dogu and deploys it into the K8s-EcoSystem. ##@ K8s - Dogu - Resource # The additional k8s yaml files K8S_RESOURCE_PRODUCTIVE_FOLDER ?= $(WORKDIR)/k8s K8S_RESOURCE_PRODUCTIVE_YAML ?= $(K8S_RESOURCE_PRODUCTIVE_FOLDER)/$(ARTIFACT_ID).yaml -K8S_RESOURCE_DOGU_CR_TEMPLATE_YAML ?= $(WORKDIR)/build/make/k8s-dogu.tpl +K8S_RESOURCE_DOGU_CR_TEMPLATE_YAML ?= $(BUILD_DIR)/make/k8s-dogu.tpl +K8S_RESOURCE_DOGU ?= $(K8S_RESOURCE_TEMP_FOLDER)/$(ARTIFACT_ID).yaml # The pre generation script creates a k8s resource yaml containing the dogu crd and the content from the k8s folder. -.PHONY: k8s-create-temporary-resource - k8s-create-temporary-resource: ${BINARY_YQ} $(K8S_RESOURCE_TEMP_FOLDER) - @echo "Generating temporary K8s resources $(K8S_RESOURCE_TEMP_YAML)..." - @rm -f $(K8S_RESOURCE_TEMP_YAML) - @sed "s|NAMESPACE|$(ARTIFACT_NAMESPACE)|g" $(K8S_RESOURCE_DOGU_CR_TEMPLATE_YAML) | sed "s|NAME|$(ARTIFACT_ID)|g" | sed "s|VERSION|$(VERSION)|g" >> $(K8S_RESOURCE_TEMP_YAML) +.PHONY: create-dogu-resource +create-dogu-resource: ${BINARY_YQ} $(K8S_RESOURCE_TEMP_FOLDER) + @echo "Generating temporary K8s resources $(K8S_RESOURCE_DOGU)..." + @rm -f $(K8S_RESOURCE_DOGU) + @sed "s|NAMESPACE|$(ARTIFACT_NAMESPACE)|g" $(K8S_RESOURCE_DOGU_CR_TEMPLATE_YAML) | sed "s|NAME|$(ARTIFACT_ID)|g" | sed "s|VERSION|$(VERSION)|g" >> $(K8S_RESOURCE_DOGU) @echo "Done." +.PHONY: apply-dogu-resource +apply-dogu-resource: + @kubectl apply -f "$(K8S_RESOURCE_DOGU)" + ##@ K8s - Dogu .PHONY: install-dogu-descriptor install-dogu-descriptor: ${BINARY_YQ} $(TARGET_DIR) ## Installs a configmap with current dogu.json into the cluster. @echo "Generate configmap from dogu.json..." - @$(BINARY_YQ) ".Image=\"${IMAGE_DEV_WITHOUT_TAG}\"" ${DOGU_JSON_FILE} > ${DOGU_JSON_DEV_FILE} + @$(BINARY_YQ) -oj ".Image=\"${IMAGE_DEV_WITHOUT_TAG}\"" ${DOGU_JSON_FILE} > ${DOGU_JSON_DEV_FILE} @kubectl create configmap "$(ARTIFACT_ID)-descriptor" --from-file=$(DOGU_JSON_DEV_FILE) --dry-run=client -o yaml | kubectl apply -f - --namespace=${NAMESPACE} @echo "Done." diff --git a/build/make/k8s-dogu.tpl b/build/make/k8s-dogu.tpl index 44cd9cf..296da65 100644 --- a/build/make/k8s-dogu.tpl +++ b/build/make/k8s-dogu.tpl @@ -3,7 +3,7 @@ kind: Dogu metadata: name: NAME labels: - dogu: NAME + app: ces spec: name: NAMESPACE/NAME version: VERSION \ No newline at end of file diff --git a/build/make/k8s.mk b/build/make/k8s.mk index 8e0fa4a..0f9fe02 100644 --- a/build/make/k8s.mk +++ b/build/make/k8s.mk @@ -7,23 +7,48 @@ endif ## Variables BINARY_YQ = $(UTILITY_BIN_PATH)/yq +BINARY_YQ_4_VERSION?=v4.40.3 +BINARY_HELM = $(UTILITY_BIN_PATH)/helm +BINARY_HELM_VERSION?=v3.13.0 +CONTROLLER_GEN = $(UTILITY_BIN_PATH)/controller-gen +CONTROLLER_GEN_VERSION?=v0.13.0 + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec # The productive tag of the image IMAGE ?= +# Set production as default stage. Use "development" as stage in your .env file to generate artifacts +# with development images pointing to K3S_CLUSTER_FQDN. +STAGE?=production K3S_CLUSTER_FQDN?=k3ces.local K3S_LOCAL_REGISTRY_PORT?=30099 K3CES_REGISTRY_URL_PREFIX="${K3S_CLUSTER_FQDN}:${K3S_LOCAL_REGISTRY_PORT}" +## Image URL to use all building/pushing image targets +IMAGE_DEV?=${K3CES_REGISTRY_URL_PREFIX}/${ARTIFACT_ID} +IMAGE_DEV_VERSION=${IMAGE_DEV}:${VERSION} # Variables for the temporary yaml files. These are used as template to generate a development resource containing # the current namespace and the dev image. -K8S_RESOURCE_TEMP_FOLDER ?= $(TARGET_DIR)/make/k8s -K8S_RESOURCE_TEMP_YAML ?= $(K8S_RESOURCE_TEMP_FOLDER)/$(ARTIFACT_ID)_$(VERSION).yaml +K8S_RESOURCE_TEMP_FOLDER ?= $(TARGET_DIR)/k8s + +# This can be used by components with own images to check if all image env var are set. +# These components should override this variable with `check-all-vars`. +CHECK_VAR_TARGETS?=check-all-vars-without-image ##@ K8s - Variables .PHONY: check-all-vars -check-all-vars: check-k8s-image-env-var check-k8s-artifact-id check-etc-hosts check-insecure-cluster-registry check-k8s-namespace-env-var ## Conduct a sanity check against selected build artefacts or local environment +check-all-vars: check-all-vars-without-image check-all-image-vars ## Conduct a sanity check against selected build artefacts or local environment + +.PHONY: check-all-image-vars +check-all-image-vars: check-k8s-image-env-var check-k8s-image-dev-var check-etc-hosts check-insecure-cluster-registry + +.PHONY: check-all-vars-without-image +check-all-vars-without-image: check-k8s-artifact-id check-k8s-namespace-env-var .PHONY: check-k8s-namespace-env-var check-k8s-namespace-env-var: @@ -52,40 +77,18 @@ check-insecure-cluster-registry: ${K8S_RESOURCE_TEMP_FOLDER}: @mkdir -p $@ -.PHONY: k8s-delete -k8s-delete: k8s-generate $(K8S_POST_GENERATE_TARGETS) ## Deletes all dogu related resources from the K8s cluster. - @echo "Delete old dogu resources..." - @kubectl delete -f $(K8S_RESOURCE_TEMP_YAML) --wait=false --ignore-not-found=true --namespace=${NAMESPACE} - -# The additional targets executed after the generate target, executed before each apply and delete. The generate target -# produces a temporary yaml. This yaml is accessible via K8S_RESOURCE_TEMP_YAML an can be changed before the apply/delete. -K8S_POST_GENERATE_TARGETS ?= -# The additional targets executed before the generate target, executed before each apply and delete. -K8S_PRE_GENERATE_TARGETS ?= k8s-create-temporary-resource - -.PHONY: k8s-generate -k8s-generate: ${BINARY_YQ} $(K8S_RESOURCE_TEMP_FOLDER) $(K8S_PRE_GENERATE_TARGETS) ## Generates the final resource yaml. - @echo "Applying general transformations..." - @sed -i "s/'{{ .Namespace }}'/$(NAMESPACE)/" $(K8S_RESOURCE_TEMP_YAML) - @$(BINARY_YQ) -i e "(select(.kind == \"Deployment\").spec.template.spec.containers[]|select(.image == \"*$(ARTIFACT_ID)*\").image)=\"$(IMAGE_DEV)\"" $(K8S_RESOURCE_TEMP_YAML) - @echo "Done." - -.PHONY: k8s-apply -k8s-apply: k8s-generate $(K8S_POST_GENERATE_TARGETS) ## Applies all generated K8s resources to the current cluster and namespace. - @echo "Apply generated K8s resources..." - @kubectl apply -f $(K8S_RESOURCE_TEMP_YAML) --namespace=${NAMESPACE} ##@ K8s - Docker .PHONY: docker-build docker-build: check-k8s-image-env-var ## Builds the docker image of the K8s app. - @echo "Building docker image..." - DOCKER_BUILDKIT=1 docker build . -t $(IMAGE) + @echo "Building docker image $(IMAGE)..." + @DOCKER_BUILDKIT=1 docker build . -t $(IMAGE) .PHONY: docker-dev-tag docker-dev-tag: check-k8s-image-dev-var docker-build ## Tags a Docker image for local K3ces deployment. - @echo "Tagging image with dev tag..." - DOCKER_BUILDKIT=1 docker tag ${IMAGE} ${IMAGE_DEV} + @echo "Tagging image with dev tag $(IMAGE_DEV_VERSION)..." + @DOCKER_BUILDKIT=1 docker tag ${IMAGE} $(IMAGE_DEV_VERSION) .PHONY: check-k8s-image-dev-var check-k8s-image-dev-var: @@ -96,8 +99,8 @@ endif .PHONY: image-import image-import: check-all-vars check-k8s-artifact-id docker-dev-tag ## Imports the currently available image into the cluster-local registry. - @echo "Import ${IMAGE_DEV} into K8s cluster ${K3S_CLUSTER_FQDN}..." - @docker push ${IMAGE_DEV} + @echo "Import $(IMAGE_DEV_VERSION) into K8s cluster ${K3S_CLUSTER_FQDN}..." + @docker push $(IMAGE_DEV_VERSION) @echo "Done." ## Functions @@ -115,5 +118,31 @@ __check_defined = \ $(if $(value $1),, \ $(error Undefined $1$(if $2, ($2)))) -${BINARY_YQ}: $(UTILITY_BIN_PATH) ## Download controller-gen locally if necessary. - $(call go-get-tool,$(BINARY_YQ),github.com/mikefarah/yq/v4@v4.25.1) \ No newline at end of file +##@ K8s - Download Utilities + +.PHONY: install-yq ## Installs the yq YAML editor. +install-yq: ${BINARY_YQ} + +${BINARY_YQ}: $(UTILITY_BIN_PATH) + $(call go-get-tool,$(BINARY_YQ),github.com/mikefarah/yq/v4@${BINARY_YQ_4_VERSION}) + +##@ K8s - Download Kubernetes Utilities + +.PHONY: install-helm ## Download helm locally if necessary. +install-helm: ${BINARY_HELM} + +${BINARY_HELM}: $(UTILITY_BIN_PATH) + $(call go-get-tool,$(BINARY_HELM),helm.sh/helm/v3/cmd/helm@${BINARY_HELM_VERSION}) + +.PHONY: controller-gen +controller-gen: ${CONTROLLER_GEN} ## Download controller-gen locally if necessary. + +${CONTROLLER_GEN}: + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_GEN_VERSION}) + +ENVTEST = $(UTILITY_BIN_PATH)/setup-envtest +.PHONY: envtest +envtest: ${ENVTEST} ## Download envtest-setup locally if necessary. + +${ENVTEST}: + $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) \ No newline at end of file diff --git a/build/make/mocks.mk b/build/make/mocks.mk index 9e61b46..4c9697f 100644 --- a/build/make/mocks.mk +++ b/build/make/mocks.mk @@ -1,14 +1,14 @@ ##@ Mocking MOCKERY_BIN=${UTILITY_BIN_PATH}/mockery -MOCKERY_VERSION=v2.20.0 +MOCKERY_VERSION?=v2.42.1 MOCKERY_YAML=${WORKDIR}/.mockery.yaml ${MOCKERY_BIN}: ${UTILITY_BIN_PATH} $(call go-get-tool,$(MOCKERY_BIN),github.com/vektra/mockery/v2@$(MOCKERY_VERSION)) ${MOCKERY_YAML}: - @cp ${WORKDIR}/build/make/mockery.yaml ${WORKDIR}/.mockery.yaml + @cp ${BUILD_DIR}/make/mockery.yaml ${WORKDIR}/.mockery.yaml .PHONY: mocks mocks: ${MOCKERY_BIN} ${MOCKERY_YAML} ## This target is used to generate mocks for all interfaces in a project. diff --git a/build/make/release.mk b/build/make/release.mk index 11dde9a..328f7ba 100644 --- a/build/make/release.mk +++ b/build/make/release.mk @@ -6,6 +6,14 @@ dogu-release: ## Start a dogu release build/make/release.sh dogu +.PHONY: node-release +node-release: ## Start a node package release + build/make/release.sh node-pkg + .PHONY: go-release go-release: ## Start a go tool release - build/make/release.sh go-tool \ No newline at end of file + build/make/release.sh go-tool + +.PHONY: dogu-cve-release +dogu-cve-release: ## Start a dogu release of a new build if the local build fixes critical CVEs + @bash -c "build/make/release_cve.sh \"${REGISTRY_USERNAME}\" \"${REGISTRY_PASSWORD}\" \"${TRIVY_IMAGE_SCAN_FLAGS}\" \"${DRY_RUN}\" \"${CVE_SEVERITY}\"" diff --git a/build/make/release.sh b/build/make/release.sh index 4fd4569..ae9a722 100755 --- a/build/make/release.sh +++ b/build/make/release.sh @@ -15,7 +15,7 @@ sourceCustomReleaseArgs() { if [[ -f "${RELEASE_ARGS_FILE}" ]]; then echo "Using custom release args file ${RELEASE_ARGS_FILE}" - sourceCustomReleaseExitCode=0 + local sourceCustomReleaseExitCode=0 # shellcheck disable=SC1090 source "${RELEASE_ARGS_FILE}" || sourceCustomReleaseExitCode=$? if [[ ${sourceCustomReleaseExitCode} -ne 0 ]]; then @@ -30,13 +30,16 @@ RELEASE_ARGS_FILE="${PROJECT_DIR}/release_args.sh" sourceCustomReleaseArgs "${RELEASE_ARGS_FILE}" +# shellcheck disable=SC1090 source "$(pwd)/build/make/release_functions.sh" TYPE="${1}" +FIXED_CVE_LIST="${2:-""}" +DRY_RUN="${3:-""}" echo "=====Starting Release process=====" -if [ "${TYPE}" == "dogu" ];then +if [[ "${TYPE}" == "dogu" || "${TYPE}" == "dogu-cve-release" ]];then CURRENT_TOOL_VERSION=$(get_current_version_by_dogu_json) else CURRENT_TOOL_VERSION=$(get_current_version_by_makefile) @@ -45,10 +48,20 @@ fi NEW_RELEASE_VERSION="$(read_new_version)" validate_new_version "${NEW_RELEASE_VERSION}" -start_git_flow_release "${NEW_RELEASE_VERSION}" +if [[ -n "${DRY_RUN}" ]]; then + start_dry_run_release "${NEW_RELEASE_VERSION}" +else + start_git_flow_release "${NEW_RELEASE_VERSION}" +fi + update_versions "${NEW_RELEASE_VERSION}" -update_changelog "${NEW_RELEASE_VERSION}" +update_changelog "${NEW_RELEASE_VERSION}" "${FIXED_CVE_LIST}" show_diff -finish_release_and_push "${CURRENT_TOOL_VERSION}" "${NEW_RELEASE_VERSION}" + +if [[ -n "${DRY_RUN}" ]]; then + abort_dry_run_release "${NEW_RELEASE_VERSION}" +else + finish_release_and_push "${CURRENT_TOOL_VERSION}" "${NEW_RELEASE_VERSION}" +fi echo "=====Finished Release process=====" diff --git a/build/make/release_cve.sh b/build/make/release_cve.sh new file mode 100755 index 0000000..6c5fc6d --- /dev/null +++ b/build/make/release_cve.sh @@ -0,0 +1,166 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + +function readCveSeverityIfUnset() { + if [ -z "${CVE_SEVERITY}" ]; then + echo "CVE_SEVERITY is unset" + while [[ -z ${CVE_SEVERITY} ]]; do + read -r -p "select the desired cve severity (CRITICAL, HIGH, MEDIUM, ...): " CVE_SEVERITY + done + fi +} + +function readCredentialsIfUnset() { + if [ -z "${USERNAME}" ]; then + echo "username is unset" + while [[ -z ${USERNAME} ]]; do + read -r -p "type username for ${REGISTRY_URL}: " USERNAME + done + fi + if [ -z "${PASSWORD}" ]; then + echo "password is unset" + while [[ -z ${PASSWORD} ]]; do + read -r -s -p "type password for ${REGISTRY_URL}: " PASSWORD + done + fi +} + +function diffArrays() { + local cveListX=("$1") + local cveListY=("$2") + local result=() + + local cveX + # Disable the following shellcheck because the arrays are sufficiently whitespace delimited because of the jq parsing result. + # shellcheck disable=SC2128 + for cveX in ${cveListX}; do + local found=0 + local cveY + for cveY in ${cveListY}; do + [[ "${cveY}" == "${cveX}" ]] && { + found=1 + break + } + done + + [[ "${found}" == 0 ]] && result+=("${cveX}") + done + + echo "${result[@]}" +} + +function dockerLogin() { + docker login "${REGISTRY_URL}" -u "${USERNAME}" -p "${PASSWORD}" +} + +function dockerLogout() { + docker logout "${REGISTRY_URL}" +} + +function nameFromDogu() { + jsonPropertyFromDogu ".Name" +} + +function imageFromDogu() { + jsonPropertyFromDogu ".Image" +} + +function versionFromDogu() { + jsonPropertyFromDogu ".Version" +} + +function jsonPropertyFromDogu() { + local property="${1}" + jq -r "${property}" "${DOGU_JSON_FILE}" +} + +function pullRemoteImage() { + docker pull "$(imageFromDogu):$(versionFromDogu)" +} + +function buildLocalImage() { + docker build --no-cache . -t "$(imageFromDogu):$(versionFromDogu)" +} + +function scanImage() { + docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS:+"${TRIVY_IMAGE_SCAN_FLAGS}"} "$(imageFromDogu):$(versionFromDogu)" +} + +function parseTrivyJsonResult() { + local severity="${1}" + local trivy_result_file="${2}" + + # First select results which have the property "Vulnerabilities". Filter the vulnerability ids with the given severity and afterward put the values in an array. + # This array is used to format the values with join(" ") in a whitespace delimited string list. + jq -rc "[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"${severity}\") | .VulnerabilityID] | unique | join(\" \")" "${trivy_result_file}" +} + +RELEASE_SH="build/make/release.sh" + +REGISTRY_URL="registry.cloudogu.com" +DOGU_JSON_FILE="dogu.json" + +CVE_SEVERITY= + +TRIVY_PATH= +TRIVY_RESULT_FILE= +TRIVY_CACHE_DIR= +TRIVY_DOCKER_CACHE_DIR=/tmp/db +TRIVY_IMAGE_SCAN_FLAGS= + +USERNAME="" +PASSWORD="" +DRY_RUN= + +function runMain() { + readCveSeverityIfUnset + readCredentialsIfUnset + dockerLogin + + mkdir -p "${TRIVY_PATH}" # Cache will not be removed after release. rm requires root because the trivy container only runs with root. + pullRemoteImage + scanImage + local remote_trivy_cve_list + remote_trivy_cve_list=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") + + buildLocalImage + scanImage + local local_trivy_cve_list + local_trivy_cve_list=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") + + dockerLogout + + local cve_in_local_but_not_in_remote + cve_in_local_but_not_in_remote=$(diffArrays "${local_trivy_cve_list}" "${remote_trivy_cve_list}") + if [[ -n "${cve_in_local_but_not_in_remote}" ]]; then + echo "Abort release. Added new vulnerabilities:" + echo "${cve_in_local_but_not_in_remote[@]}" + exit 2 + fi + + local cve_in_remote_but_not_in_local + cve_in_remote_but_not_in_local=$(diffArrays "${remote_trivy_cve_list}" "${local_trivy_cve_list}") + if [[ -z "${cve_in_remote_but_not_in_local}" ]]; then + echo "Abort release. Fixed no new vulnerabilities" + exit 3 + fi + + echo "Fixed ${CVE_SEVERITY} CVEs: ${cve_in_remote_but_not_in_local}" + "${RELEASE_SH}" "dogu-cve-release" "${cve_in_remote_but_not_in_local}" "${DRY_RUN}" +} + +# make the script only runMain when executed, not when sourced from bats tests +if [[ -n "${BASH_VERSION}" && "${BASH_SOURCE[0]}" == "${0}" ]]; then + USERNAME="${1:-""}" + PASSWORD="${2:-""}" + TRIVY_IMAGE_SCAN_FLAGS="${3:-""}" + DRY_RUN="${4:-""}" + CVE_SEVERITY="${5:-""}" + + TRIVY_PATH="/tmp/trivy-dogu-cve-release-$(nameFromDogu)" + TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + runMain +fi diff --git a/build/make/release_functions.sh b/build/make/release_functions.sh index 528806b..499c248 100755 --- a/build/make/release_functions.sh +++ b/build/make/release_functions.sh @@ -3,15 +3,15 @@ set -o errexit set -o nounset set -o pipefail -wait_for_ok(){ +wait_for_ok() { printf "\n" - OK=false - while [[ ${OK} != "ok" ]] ; do + local OK="false" + while [[ "${OK}" != "ok" ]]; do read -r -p "${1} (type 'ok'): " OK done } -ask_yes_or_no(){ +ask_yes_or_no() { local ANSWER="" while [ "${ANSWER}" != "y" ] && [ "${ANSWER}" != "n" ]; do @@ -21,49 +21,51 @@ ask_yes_or_no(){ echo "${ANSWER}" } -get_current_version_by_makefile(){ +get_current_version_by_makefile() { grep '^VERSION=[0-9[:alpha:].-]*$' Makefile | sed s/VERSION=//g } -get_current_version_by_dogu_json(){ +get_current_version_by_dogu_json() { jq ".Version" --raw-output dogu.json } -read_new_version(){ +read_new_version() { local NEW_RELEASE_VERSION read -r -p "Current Version is v${CURRENT_TOOL_VERSION}. Please provide the new version: v" NEW_RELEASE_VERSION echo "${NEW_RELEASE_VERSION}" } -validate_new_version(){ +validate_new_version() { local NEW_RELEASE_VERSION="${1}" # Validate that release version does not start with vv if [[ ${NEW_RELEASE_VERSION} = v* ]]; then echo "WARNING: The new release version (v${NEW_RELEASE_VERSION}) starts with 'vv'." echo "You must not enter the v when defining the new version." + local ANSWER ANSWER=$(ask_yes_or_no "Should the first v be removed?") if [ "${ANSWER}" == "y" ]; then NEW_RELEASE_VERSION="${NEW_RELEASE_VERSION:1}" echo "Release version now is: ${NEW_RELEASE_VERSION}" fi - fi; + fi } -start_git_flow_release(){ +start_git_flow_release() { local NEW_RELEASE_VERSION="${1}" # Do gitflow git flow init --defaults --force + local mainBranchExists mainBranchExists="$(git show-ref refs/remotes/origin/main || echo "")" if [ -n "$mainBranchExists" ]; then - echo 'Using "main" branch for production releases' - git flow config set master main - git checkout main - git pull origin main + echo 'Using "main" branch for production releases' + git flow config set master main + git checkout main + git pull origin main else - echo 'Using "master" branch for production releases' - git checkout master - git pull origin master + echo 'Using "master" branch for production releases' + git checkout master + git pull origin master fi git checkout develop @@ -71,17 +73,30 @@ start_git_flow_release(){ git flow release start v"${NEW_RELEASE_VERSION}" } +start_dry_run_release() { + local NEW_RELEASE_VERSION="${1}" + + git checkout -b dryrun/v"${NEW_RELEASE_VERSION}" +} + +abort_dry_run_release() { + local NEW_RELEASE_VERSION="${1}" + + git checkout develop + git branch -D dryrun/v"${NEW_RELEASE_VERSION}" +} + # update_versions updates files with the new release version and interactively asks the user for verification. If okay # the updated files will be staged to git and finally committed. # # extension points: # - update_versions_modify_files - update a file with the new version number # - update_versions_stage_modified_files - stage a modified file to prepare the file for the up-coming commit -update_versions(){ +update_versions() { local NEW_RELEASE_VERSION="${1}" if [[ $(type -t update_versions_modify_files) == function ]]; then - preSkriptExitCode=0 + local preSkriptExitCode=0 update_versions_modify_files "${NEW_RELEASE_VERSION}" || preSkriptExitCode=$? if [[ ${preSkriptExitCode} -ne 0 ]]; then echo "ERROR: custom update_versions_modify_files() exited with exit code ${preSkriptExitCode}" @@ -92,7 +107,7 @@ update_versions(){ # Update version in dogu.json if [ -f "dogu.json" ]; then echo "Updating version in dogu.json..." - jq ".Version = \"${NEW_RELEASE_VERSION}\"" dogu.json > dogu2.json && mv dogu2.json dogu.json + jq ".Version = \"${NEW_RELEASE_VERSION}\"" dogu.json >dogu2.json && mv dogu2.json dogu.json fi # Update version in Dockerfile @@ -110,7 +125,7 @@ update_versions(){ # Update version in package.json if [ -f "package.json" ]; then echo "Updating version in package.json..." - jq ".version = \"${NEW_RELEASE_VERSION}\"" package.json > package2.json && mv package2.json package.json + jq ".version = \"${NEW_RELEASE_VERSION}\"" package.json >package2.json && mv package2.json package.json fi # Update version in pom.xml @@ -133,7 +148,7 @@ update_versions(){ fi if [ -f "dogu.json" ]; then - git add dogu.json + git add dogu.json fi if [ -f "Dockerfile" ]; then @@ -155,12 +170,14 @@ update_versions(){ git commit -m "Bump version" } -update_changelog(){ +update_changelog() { local NEW_RELEASE_VERSION="${1}" + local FIXED_CVE_LIST="${2}" # Changelog update + local CURRENT_DATE CURRENT_DATE=$(date --rfc-3339=date) - NEW_CHANGELOG_TITLE="## [v${NEW_RELEASE_VERSION}] - ${CURRENT_DATE}" + local NEW_CHANGELOG_TITLE="## [v${NEW_RELEASE_VERSION}] - ${CURRENT_DATE}" # Check if "Unreleased" tag exists while ! grep --silent "## \[Unreleased\]" CHANGELOG.md; do echo "" @@ -169,6 +186,10 @@ update_changelog(){ wait_for_ok "Please insert a \"## [Unreleased]\" line into CHANGELOG.md now." done + if [[ -n "${FIXED_CVE_LIST}" ]]; then + addFixedCVEListFromReRelease "${FIXED_CVE_LIST}" + fi + # Add new title line to changelog sed -i "s|## \[Unreleased\]|## \[Unreleased\]\n\n${NEW_CHANGELOG_TITLE}|g" CHANGELOG.md @@ -186,8 +207,36 @@ update_changelog(){ git commit -m "Update changelog" } -show_diff(){ - if ! git diff --exit-code > /dev/null; then +# addFixedCVEListFromReRelease is used in dogu cve releases. The method adds the fixed CVEs under the ### Fixed header +# in the unreleased section. +addFixedCVEListFromReRelease() { + local fixed_cve_list="${1}" + + local cve_sed_search="" + local cve_sed_replace="" + local fixed_exists_in_unreleased + fixed_exists_in_unreleased=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^### Fixed$" || true) + if [[ -n "${fixed_exists_in_unreleased}" ]]; then + # extend fixed header with CVEs. + cve_sed_search="^\#\#\# Fixed$" + cve_sed_replace="\#\#\# Fixed\n- Fixed ${fixed_cve_list}" + else + # extend unreleased header with fixed header and CVEs. + cve_sed_search="^\#\# \[Unreleased\]$" + cve_sed_replace="\#\# \[Unreleased\]\n\#\#\# Fixed\n- Fixed ${fixed_cve_list}" + + local any_exists_unreleased + any_exists_unreleased=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^\#\#\# Added$" -e "^\#\#\# Fixed$" -e "^\#\#\# Changed$" || true) + if [[ -n ${any_exists_unreleased} ]]; then + cve_sed_replace+="\n" + fi + fi + + sed -i "0,/${cve_sed_search}/s//${cve_sed_replace}/" CHANGELOG.md +} + +show_diff() { + if ! git diff --exit-code >/dev/null; then echo "There are still uncommitted changes:" echo "" echo "# # # # # # # # # #" @@ -206,7 +255,7 @@ show_diff(){ echo "# # # # # # # # # #" } -finish_release_and_push(){ +finish_release_and_push() { local CURRENT_VERSION="${1}" local NEW_RELEASE_VERSION="${2}" diff --git a/build/make/yarn.mk b/build/make/yarn.mk index 12792b4..6ff7de9 100644 --- a/build/make/yarn.mk +++ b/build/make/yarn.mk @@ -5,14 +5,6 @@ YARN_LOCK=$(WORKDIR)/yarn.lock .PHONY: yarn-install yarn-install: $(YARN_TARGET) ## Execute yarn install -ifeq ($(ENVIRONMENT), ci) - -$(YARN_TARGET): $(YARN_LOCK) - @echo "Yarn install on CI server" - @yarn install - -else - $(YARN_TARGET): $(YARN_LOCK) $(PASSWD) @echo "Executing yarn..." @docker run --rm \ @@ -24,4 +16,24 @@ $(YARN_TARGET): $(YARN_LOCK) $(PASSWD) yarn install @touch $@ -endif +.PHONY yarn-publish-ci: +yarn-publish-ci: ## Execute yarn publish with '--non-interactive' flag to suppress the version prompt + @echo "Executing yarn publish..." + @docker run --rm \ + -u "$(UID_NR):$(GID_NR)" \ + -v $(PASSWD):/etc/passwd:ro \ + -v $(WORKDIR):$(WORKDIR) \ + -w $(WORKDIR) \ + node:$(NODE_VERSION) \ + yarn publish --non-interactive + +.PHONY yarn-publish: ## Execute yarn publish +yarn-publish: $(YARN_BUILD_TARGET) + @echo "Executing yarn publish..." + @docker run --rm \ + -u "$(UID_NR):$(GID_NR)" \ + -v $(PASSWD):/etc/passwd:ro \ + -v $(WORKDIR):$(WORKDIR) \ + -w $(WORKDIR) \ + node:$(NODE_VERSION) \ + yarn publish diff --git a/installation-scripts/install-mysql.sh b/installation-scripts/install-mysql.sh index 1efbd81..1e5ab5f 100755 --- a/installation-scripts/install-mysql.sh +++ b/installation-scripts/install-mysql.sh @@ -16,6 +16,7 @@ dpkg -i mysql-apt-config_0.8.18-1_all.deb < Date: Mon, 8 Apr 2024 13:44:08 +0200 Subject: [PATCH 2/6] #17 update Changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a9b86..96b84c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Upgrade Makefiles to 9.0.3 +- Upgrade packages to fix CVE-2023-25775 & CVE-2023-5178 ## [v8.0.33-2] - 2023-06-27 ### Added From 1c191547e08114071155b83d45591c6a45ead0b3 Mon Sep 17 00:00:00 2001 From: meiserloh Date: Mon, 8 Apr 2024 14:12:53 +0200 Subject: [PATCH 3/6] #17 change mysql public key --- installation-scripts/install-mysql.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/installation-scripts/install-mysql.sh b/installation-scripts/install-mysql.sh index 1e5ab5f..37374cf 100755 --- a/installation-scripts/install-mysql.sh +++ b/installation-scripts/install-mysql.sh @@ -15,7 +15,6 @@ dpkg -i mysql-apt-config_0.8.18-1_all.deb < Date: Mon, 8 Apr 2024 14:39:48 +0200 Subject: [PATCH 4/6] #17 add comment with website for the public key to install script --- installation-scripts/install-mysql.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/installation-scripts/install-mysql.sh b/installation-scripts/install-mysql.sh index 37374cf..d6f5b5e 100755 --- a/installation-scripts/install-mysql.sh +++ b/installation-scripts/install-mysql.sh @@ -15,6 +15,7 @@ dpkg -i mysql-apt-config_0.8.18-1_all.deb < Date: Tue, 9 Apr 2024 14:39:50 +0200 Subject: [PATCH 5/6] Bump version --- Dockerfile | 2 +- dogu.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 813004d..b99c1a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM registry.cloudogu.com/official/base-debian:11.6-1 LABEL MAINTAINER="hello@cloudogu.com" \ NAME="official/mysql" \ - VERSION="8.0.33-2" + VERSION="8.0.33-3" ENV PATH="${PATH}:/var/lib/mysql/bin" \ MYSQL_VOLUME=/var/lib/mysql \ diff --git a/dogu.json b/dogu.json index 2291fba..c76d8ef 100644 --- a/dogu.json +++ b/dogu.json @@ -1,6 +1,6 @@ { "Name": "official/mysql", - "Version": "8.0.33-2", + "Version": "8.0.33-3", "DisplayName": "MySQL", "Description": "MySQL - Relational database", "Url": "https://www.mysql.com/", From 265a9b866de4ab37e1a0d030b258b6956eff74dc Mon Sep 17 00:00:00 2001 From: meiserloh Date: Tue, 9 Apr 2024 14:40:23 +0200 Subject: [PATCH 6/6] Update changelog --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96b84c7..15e9a35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [v8.0.33-3] - 2024-04-09 +### Fixed +- Fixed CVE-2023-25775 CVE-2023-5178 + ### Changed - Upgrade Makefiles to 9.0.3 -- Upgrade packages to fix CVE-2023-25775 & CVE-2023-5178 ## [v8.0.33-2] - 2023-06-27 ### Added