From a9d62ea085fe30ac5836b738635f11f02438088a Mon Sep 17 00:00:00 2001 From: Jackson West Date: Wed, 11 Oct 2023 23:34:48 -0500 Subject: [PATCH] refactors logging to use annonation style --- Common.mk | 96 +++++++++---------- build/lib/common.sh | 10 +- build/lib/create_attribution.sh | 2 + build/lib/make_shell.sh | 54 +++++++++++ build/lib/make_shell_trace.sh | 23 ----- projects/containerd/containerd/Makefile | 6 +- .../kubernetes-sigs/image-builder/Makefile | 60 +++++------- projects/kubernetes-sigs/kind/Makefile | 14 +-- 8 files changed, 138 insertions(+), 127 deletions(-) create mode 100755 build/lib/make_shell.sh delete mode 100755 build/lib/make_shell_trace.sh diff --git a/Common.mk b/Common.mk index 6706f88b78..138fd216a4 100644 --- a/Common.mk +++ b/Common.mk @@ -13,8 +13,8 @@ BUILD_LIB=$(BASE_DIRECTORY)/build/lib OUTPUT_BIN_DIR?=$(OUTPUT_DIR)/bin/$(REPO) SHELL_TRACE?=false -SHELL:=$(if $(filter true,$(SHELL_TRACE)),$(BUILD_LIB)/make_shell_trace.sh,bash) -.SHELLFLAGS:=$(if $(filter true,$(SHELL_TRACE)),-c,-eu -o pipefail -c) +DEFAULT_SHELL:=$(if $(filter true,$(SHELL_TRACE)),$(BUILD_LIB)/make_shell.sh trace,bash) +.SHELLFLAGS:=-eu -o pipefail -c #################### AWS ########################### AWS_REGION?=us-west-2 AWS_ACCOUNT_ID?=$(shell aws sts get-caller-identity --query Account --output text) @@ -439,6 +439,17 @@ DATE_CMD=TZ=utc $(shell source $(BUILD_LIB)/common.sh && build::find::gnu_varian DATE_NANO=$(shell if [ "$$(uname -s)" = "Linux" ] || command -v gdate &> /dev/null; then echo %3N; fi) TARGET_START_LOG?=$(eval _START_TIME:=$(shell $(DATE_CMD) +%s.$(DATE_NANO)))\\n------------------- $(shell $(DATE_CMD) +"%Y-%m-%dT%H:%M:%S.$(DATE_NANO)%z") Starting target=$@ ------------------- TARGET_END_LOG?="------------------- `$(DATE_CMD) +'%Y-%m-%dT%H:%M:%S.$(DATE_NANO)%z'` Finished target=$@ duration=`echo $$($(DATE_CMD) +%s.$(DATE_NANO)) - $(_START_TIME) | bc` seconds -------------------\\n" + +ENABLE_LOGGING=$(eval $(call ENABLE_LOGGING_BODY,$@)) +# $1 - target name +# target is exported so that the script can tell the difference between +# a recipe and a $(shell) call, which we do not want to log around +# this style of enable logging only works for single line recipes +define ENABLE_LOGGING_BODY +$(eval _TARGET:=$1) +$(_TARGET): export LOGGING_TARGET=$(_TARGET) +$(_TARGET): SHELL=$(BUILD_LIB)/make_shell.sh log +endef #################################################### #################### TARGETS FOR OVERRIDING ######## @@ -477,14 +488,14 @@ define CGO_DOCKER endef define SIMPLE_CREATE_BINARIES_SHELL - $(BASE_DIRECTORY)/build/lib/simple_create_binaries.sh $(MAKE_ROOT) $(MAKE_ROOT)/$(OUTPUT_PATH) $(REPO) $(GOLANG_VERSION) $(PLATFORM) "$(SOURCE_PATTERN)" \ + @$(BASE_DIRECTORY)/build/lib/simple_create_binaries.sh $(MAKE_ROOT) $(MAKE_ROOT)/$(OUTPUT_PATH) $(REPO) $(GOLANG_VERSION) $(PLATFORM) "$(SOURCE_PATTERN)" \ "$(GOBUILD_COMMAND)" "$(EXTRA_GOBUILD_FLAGS)" "$(GO_LDFLAGS)" $(CGO_ENABLED) "$(CGO_LDFLAGS)" "$(GO_MOD_PATH)" "$(BINARY_TARGET_FILES_BUILD_TOGETHER)" endef # $1 - make target # $2 - target directory define CGO_CREATE_BINARIES_SHELL - $(MAKE) binary-builder/cgo/$(PLATFORM:linux/%=%) IMAGE_OUTPUT=dest=$(OUTPUT_BIN_DIR)/$(2) CGO_TARGET=$(1) IMAGE_BUILD_ARGS="GOPROXY COMPONENT CGO_TARGET" + @$(MAKE) binary-builder/cgo/$(PLATFORM:linux/%=%) IMAGE_OUTPUT=dest=$(OUTPUT_BIN_DIR)/$(2) CGO_TARGET=$(1) IMAGE_BUILD_ARGS="GOPROXY COMPONENT CGO_TARGET" endef define WRITE_LOCAL_IMAGE_TAG @@ -572,10 +583,8 @@ $(OUTPUT_BIN_DIR)/%: BINARY_TARGET=$(@F:%.exe=%) $(OUTPUT_BIN_DIR)/%: SOURCE_PATTERN=$(if $(filter $(BINARY_TARGET),$(BINARY_TARGET_FILES_BUILD_TOGETHER)),$(SOURCE_PATTERNS_BUILD_TOGETHER),$(word $(call pos,$(BINARY_TARGET),$(BINARY_TARGET_FILES)),$(SOURCE_PATTERNS))) $(OUTPUT_BIN_DIR)/%: OUTPUT_PATH=$(if $(and $(if $(filter false,$(call IS_ONE_WORD,$(BINARY_TARGET_FILES_BUILD_TOGETHER))),$(filter $(BINARY_TARGET),$(BINARY_TARGET_FILES_BUILD_TOGETHER)))),$(@D)/,$@) $(OUTPUT_BIN_DIR)/%: GO_MOD_PATH=$($(call GO_MOD_TARGET_FOR_BINARY_VAR_NAME,$(BINARY_TARGET))) -$(OUTPUT_BIN_DIR)/%: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) - @echo -e $(call TARGET_START_LOG) +$(OUTPUT_BIN_DIR)/%: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) | $$(ENABLE_LOGGING) $(if $(NEEDS_CGO_BUILDER),$(call CGO_CREATE_BINARIES_SHELL,$@,$(*D)),$(call SIMPLE_CREATE_BINARIES_SHELL)) - @echo -e $(call TARGET_END_LOG) endif .PHONY: binaries @@ -619,10 +628,8 @@ $(OUTPUT_DIR)/%TTRIBUTION.txt: $(OUTPUT_DIR)/%ttribution/go-license.csv: BINARY_TARGET=$(if $(filter .,$(*D)),,$(*D)) $(OUTPUT_DIR)/%ttribution/go-license.csv: GO_MOD_PATH=$(if $(BINARY_TARGET),$(GO_MOD_TARGET_FOR_BINARY_$(call TO_UPPER,$(BINARY_TARGET))),$(word 1,$(UNIQ_GO_MOD_PATHS))) $(OUTPUT_DIR)/%ttribution/go-license.csv: LICENSE_PACKAGE_FILTER=$(GO_MOD_$(subst /,_,$(GO_MOD_PATH))_LICENSE_PACKAGE_FILTER) -$(OUTPUT_DIR)/%ttribution/go-license.csv: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) | ensure-jq - @echo -e $(call TARGET_START_LOG) - $(BASE_DIRECTORY)/build/lib/gather_licenses.sh $(REPO) $(MAKE_ROOT)/$(OUTPUT_DIR)/$(BINARY_TARGET) "$(LICENSE_PACKAGE_FILTER)" $(GO_MOD_PATH) $(GOLANG_VERSION) $(LICENSE_THRESHOLD) - @echo -e $(call TARGET_END_LOG) +$(OUTPUT_DIR)/%ttribution/go-license.csv: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) | ensure-jq $$(ENABLE_LOGGING) + @$(BASE_DIRECTORY)/build/lib/gather_licenses.sh $(REPO) $(MAKE_ROOT)/$(OUTPUT_DIR)/$(BINARY_TARGET) "$(LICENSE_PACKAGE_FILTER)" $(GO_MOD_PATH) $(GOLANG_VERSION) $(LICENSE_THRESHOLD) .PHONY: gather-licenses gather-licenses: $(GATHER_LICENSES_TARGETS) @@ -631,20 +638,15 @@ gather-licenses: $(GATHER_LICENSES_TARGETS) # if there is only one go mod path so only one attribution is created, the file will be named ATTRIBUTION.txt and licenses will be stored in _output, `%` will equal `A` # if multiple attributions are being generated, the file will be _ATTRIBUTION.txt and licenses will be stored in _output/, `%` will equal `_A` %TTRIBUTION.txt: LICENSE_OUTPUT_PATH=$(OUTPUT_DIR)$(if $(filter A,$(*F)),,/$(call TO_LOWER,$(*F:%_A=%))) -%TTRIBUTION.txt: $$(LICENSE_OUTPUT_PATH)/attribution/go-license.csv - @echo -e $(call TARGET_START_LOG) - @rm -f $(@F) - $(BASE_DIRECTORY)/build/lib/create_attribution.sh $(MAKE_ROOT) $(GOLANG_VERSION) $(MAKE_ROOT)/$(LICENSE_OUTPUT_PATH) $(@F) $(RELEASE_BRANCH) - @echo -e $(call TARGET_END_LOG) +%TTRIBUTION.txt: $$(LICENSE_OUTPUT_PATH)/attribution/go-license.csv | $$(ENABLE_LOGGING) + @$(BASE_DIRECTORY)/build/lib/create_attribution.sh $(MAKE_ROOT) $(GOLANG_VERSION) $(MAKE_ROOT)/$(LICENSE_OUTPUT_PATH) $(@F) $(RELEASE_BRANCH) .PHONY: attribution attribution: $(and $(filter true,$(HAS_LICENSES)),$(ATTRIBUTION_TARGETS)) .PHONY: attribution-pr -attribution-pr: attribution - @echo -e $(call TARGET_START_LOG) - $(BASE_DIRECTORY)/build/update-attribution-files/create_pr.sh - @echo -e $(call TARGET_END_LOG) +attribution-pr: attribution | $$(ENABLE_LOGGING) + @$(BASE_DIRECTORY)/build/update-attribution-files/create_pr.sh .PHONY: all-attributions all-attributions: @@ -653,18 +655,14 @@ all-attributions: #### Tarball Targets .PHONY: tarballs -tarballs: $(LICENSES_TARGETS_FOR_PREREQ) +tarballs: $(LICENSES_TARGETS_FOR_PREREQ) | $$(ENABLE_LOGGING) ifeq ($(SIMPLE_CREATE_TARBALLS),true) - @echo -e $(call TARGET_START_LOG) - $(BASE_DIRECTORY)/build/lib/simple_create_tarballs.sh $(TAR_FILE_PREFIX) $(MAKE_ROOT)/$(OUTPUT_DIR) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) $(GIT_TAG) "$(BINARY_PLATFORMS)" $(ARTIFACTS_PATH) $(GIT_HASH) - @echo -e $(call TARGET_END_LOG) + @$(BUILD_LIB)/simple_create_tarballs.sh $(TAR_FILE_PREFIX) $(MAKE_ROOT)/$(OUTPUT_DIR) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) $(GIT_TAG) "$(BINARY_PLATFORMS)" $(ARTIFACTS_PATH) $(GIT_HASH) endif .PHONY: upload-artifacts -upload-artifacts: s3-artifacts upload-output-to-prow-artifacts-s3-artifacts - @echo -e $(call TARGET_START_LOG) - $(BASE_DIRECTORY)/build/lib/upload_artifacts.sh $(ARTIFACTS_PATH) $(ARTIFACTS_BUCKET) $(ARTIFACTS_UPLOAD_PATH) $(BUILD_IDENTIFIER) $(GIT_HASH) $(LATEST) $(UPLOAD_DRY_RUN) $(UPLOAD_DO_NOT_DELETE) - @echo -e $(call TARGET_END_LOG) +upload-artifacts: s3-artifacts upload-output-to-prow-artifacts-s3-artifacts | $$(ENABLE_LOGGING) + @$(BUILD_LIB)/upload_artifacts.sh $(ARTIFACTS_PATH) $(ARTIFACTS_BUCKET) $(ARTIFACTS_UPLOAD_PATH) $(BUILD_IDENTIFIER) $(GIT_HASH) $(LATEST) $(UPLOAD_DRY_RUN) $(UPLOAD_DO_NOT_DELETE) .PHONY: s3-artifacts s3-artifacts: tarballs @@ -682,19 +680,15 @@ upload-output-to-prow-artifacts-%: ### Checksum Targets .PHONY: checksums -checksums: $(BINARY_TARGETS) +checksums: $(BINARY_TARGETS) | $$(ENABLE_LOGGING) ifneq ($(strip $(BINARY_TARGETS)),) - @echo -e $(call TARGET_START_LOG) - $(BASE_DIRECTORY)/build/lib/update_checksums.sh $(MAKE_ROOT) $(PROJECT_ROOT) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) - @echo -e $(call TARGET_END_LOG) + @$(BASE_DIRECTORY)/build/lib/update_checksums.sh $(MAKE_ROOT) $(PROJECT_ROOT) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) endif .PHONY: validate-checksums -validate-checksums: $(BINARY_TARGETS) upload-output-to-prow-artifacts-validate-checksums +validate-checksums: $(BINARY_TARGETS) upload-output-to-prow-artifacts-validate-checksums | $$(ENABLE_LOGGING) ifneq ($(and $(strip $(BINARY_TARGETS)), $(filter false, $(SKIP_CHECKSUM_VALIDATION))),) - @echo -e $(call TARGET_START_LOG) - $(BASE_DIRECTORY)/build/lib/validate_checksums.sh $(MAKE_ROOT) $(PROJECT_ROOT) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) $(FAKE_ARM_BINARIES_FOR_VALIDATION) $(FAKE_AMD_BINARIES_FOR_VALIDATION) - @echo -e $(call TARGET_END_LOG) + @$(BASE_DIRECTORY)/build/lib/validate_checksums.sh $(MAKE_ROOT) $(PROJECT_ROOT) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) $(FAKE_ARM_BINARIES_FOR_VALIDATION) $(FAKE_AMD_BINARIES_FOR_VALIDATION) endif .PHONY: attribution-checksums @@ -742,10 +736,8 @@ clean-job-caches: $(and $(findstring presubmit,$(JOB_TYPE)),$(filter true,$(PRUN %/images/amd64 %/images/arm64: IMAGE_OUTPUT_TYPE?=oci %/images/amd64 %/images/arm64: IMAGE_OUTPUT?=dest=$(IMAGE_OUTPUT_DIR)/$(IMAGE_OUTPUT_NAME).tar -%/images/push: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) | ensure-buildctl ensure-buildkitd-host - @echo -e $(call TARGET_START_LOG) - $(BUILDCTL) - @echo -e $(call TARGET_END_LOG) +%/images/push: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) | ensure-buildctl ensure-buildkitd-host $$(ENABLE_LOGGING) + @$(BUILDCTL) %/images/amd64: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) | ensure-buildctl ensure-buildkitd-host @echo -e $(call TARGET_START_LOG) @@ -796,10 +788,8 @@ binary-builder/cgo/%: USE_DOCKER_FOR_CGO_BUILD=$(shell command -v docker &> /dev ## Helm Targets .PHONY: helm/pull -helm/pull: - @echo -e $(call TARGET_START_LOG) - $(BUILD_LIB)/helm_pull.sh $(HELM_PULL_LOCATION) $(HELM_REPO_URL) $(HELM_PULL_NAME) $(REPO) $(HELM_DIRECTORY) $(CHART_VERSION) $(COPY_CRDS) - @echo -e $(call TARGET_END_LOG) +helm/pull: | $$(ENABLE_LOGGING) + @$(BUILD_LIB)/helm_pull.sh $(HELM_PULL_LOCATION) $(HELM_REPO_URL) $(HELM_PULL_NAME) $(REPO) $(HELM_DIRECTORY) $(CHART_VERSION) $(COPY_CRDS) # Build helm chart .PHONY: helm/build @@ -815,19 +805,15 @@ helm/build: $(if $(wildcard $(PROJECT_ROOT)/helm/patches),$(HELM_GIT_PATCH_TARGE # Build helm chart and push to registry defined in IMAGE_REPO. .PHONY: helm/push -helm/push: helm/build | ensure-helm - @echo -e $(call TARGET_START_LOG) - $(BUILD_LIB)/helm_push.sh $(IMAGE_REPO) $(HELM_DESTINATION_REPOSITORY) $(HELM_TAG) $(GIT_TAG) $(OUTPUT_DIR) $(LATEST) - @echo -e $(call TARGET_END_LOG) +helm/push: helm/build | ensure-helm $$(ENABLE_LOGGING) + @$(BUILD_LIB)/helm_push.sh $(IMAGE_REPO) $(HELM_DESTINATION_REPOSITORY) $(HELM_TAG) $(GIT_TAG) $(OUTPUT_DIR) $(LATEST) ## Fetch Binary Targets .PHONY: handle-dependencies handle-dependencies: $(call PROJECT_DEPENDENCIES_TARGETS) -$(BINARY_DEPS_DIR)/linux-%: - @echo -e $(call TARGET_START_LOG) +$(BINARY_DEPS_DIR)/linux-%: | $$(ENABLE_LOGGING) $(BUILD_LIB)/fetch_binaries.sh $(BINARY_DEPS_DIR) $* $(ARTIFACTS_BUCKET) $(LATEST) $(RELEASE_BRANCH) - @echo -e $(call TARGET_END_LOG) ## Build Targets .PHONY: build @@ -840,7 +826,8 @@ release: $(RELEASE_TARGETS) # Iterate over release branch versions, avoiding branches explicitly marked as skipped .PHONY: %/release-branches/all %/release-branches/all: - @for version in $(SUPPORTED_K8S_VERSIONS) ; do \ + @set -e; \ + for version in $(SUPPORTED_K8S_VERSIONS) ; do \ if ! [[ "$(SKIPPED_K8S_VERSIONS)" =~ $$version ]]; then \ $(MAKE) $* $(if $(filter true,$(BINARIES_ARE_RELEASE_BRANCHED)),clean-output,) RELEASE_BRANCH=$$version; \ fi \ @@ -1062,3 +1049,8 @@ run-$(1)-in-docker: run-target-in-docker endef $(foreach target,$(IN_DOCKER_TARGETS),$(eval $(call RUN_IN_DOCKER_TARGET,$(target)))) + +# make sure by default all targets use the +# if we do not have this as a catch all target then the first target +# which sets it to 0 will affect all the prereq targets for that target +%: SHELL=$(DEFAULT_SHELL) diff --git a/build/lib/common.sh b/build/lib/common.sh index 5a90b14560..715eadff00 100755 --- a/build/lib/common.sh +++ b/build/lib/common.sh @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Short-circuit if script has already been sourced +[[ $(type -t build::common::loaded) == function ]] && return 0 + BUILD_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/" && pwd -P)" source "${BUILD_ROOT}/eksd_releases.sh" @@ -149,7 +152,7 @@ function build::gather_licenses() { # data about each dependency to generate the amazon approved attribution.txt files # go-deps is needed for module versions # go-licenses are all the dependencies found from the module(s) that were passed in via patterns - build::common::echo_and_run go list -deps=true -json ./... | jq -s '' > "${outputdir}/attribution/go-deps.json" + build::common::echo_and_run go list -deps=true -json ./... | jq -s '.' > "${outputdir}/attribution/go-deps.json" # go-licenses can be a bit noisy with its output and lot of it can be confusing # the following messages are safe to ignore since we do not need the license url for our process @@ -454,3 +457,8 @@ function build::common::copy_if_source_destination_different() { cp -rf $source $destination } + +# Marker function to indicate script has been fully sourced +function build::common::loaded() { + return 0 +} diff --git a/build/lib/create_attribution.sh b/build/lib/create_attribution.sh index 5eeb15fc25..062fd91069 100755 --- a/build/lib/create_attribution.sh +++ b/build/lib/create_attribution.sh @@ -26,6 +26,8 @@ OUTPUT_DIR="$3" OUTPUT_FILENAME="$4" RELEASE_BRANCH="${5:-}" +rm -f $OUTPUT_FILENAME + if [[ -n "$RELEASE_BRANCH" ]] && [ -d "$PROJECT_ROOT/$RELEASE_BRANCH" ]; then PROJECT_ROOT=$PROJECT_ROOT/$RELEASE_BRANCH fi diff --git a/build/lib/make_shell.sh b/build/lib/make_shell.sh new file mode 100755 index 0000000000..60a042c22d --- /dev/null +++ b/build/lib/make_shell.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# args: <-c|-eu -o pipefail -c> + +ACTION="$1" +if [ "$ACTION" = "trace" ]; then + >&2 echo "Shell trace: $@" +fi + +# remove action and shellflags up to the -c +for var; do + shift + [ "$var" = '-c' ] && break; +done + +if [ -z "${LOGGING_TARGET:-}" ] || [ "$ACTION" = "trace" ]; then + eval "$@" + exit $? +fi + +# in case of recursive make calls, unset the TARGET env var to avoid +# logging in the child call when not desired +TARGET=$LOGGING_TARGET +unset LOGGING_TARGET + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +source "${SCRIPT_ROOT}/common.sh" + +DATE=$(build::find::gnu_variant_on_mac date) +DATE_NANO=$(if [ "$(uname -s)" = "Linux" ] || [ "$DATE" = "gdate" ]; then echo %3N; fi) + +START_TIME=$($DATE +%s.$DATE_NANO) + +echo -e "\n------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") Starting target=$TARGET -------------------" +echo "($(pwd)) \$ $@" +eval "$@" +echo -e "------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") Finished target=$TARGET duration=$(echo $($DATE +%s.$DATE_NANO) - $START_TIME | bc) seconds -------------------\n" diff --git a/build/lib/make_shell_trace.sh b/build/lib/make_shell_trace.sh deleted file mode 100755 index 6a5dc6b76a..0000000000 --- a/build/lib/make_shell_trace.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -# remove -c passed by make -shift ->&2 echo "Shell trace: $@" -eval "$@" diff --git a/projects/containerd/containerd/Makefile b/projects/containerd/containerd/Makefile index ae6a3e2ad8..730cdc3668 100644 --- a/projects/containerd/containerd/Makefile +++ b/projects/containerd/containerd/Makefile @@ -34,10 +34,8 @@ SIMPLE_CREATE_TARBALLS=false include $(BASE_DIRECTORY)/Common.mk .PHONY: tarballs -tarballs: $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) - @echo -e $(call TARGET_START_LOG) - build/create_tarballs.sh $(TAR_FILE_PREFIX) $(MAKE_ROOT)/$(OUTPUT_DIR) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) $(GIT_TAG) "$(BINARY_PLATFORMS)" $(ARTIFACTS_PATH) $(GIT_HASH) $(MAKE_ROOT)/$(BINARY_DEPS_DIR) - @echo -e $(call TARGET_END_LOG) +tarballs: $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) | $$(ENABLE_LOGGING) + @build/create_tarballs.sh $(TAR_FILE_PREFIX) $(MAKE_ROOT)/$(OUTPUT_DIR) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) $(GIT_TAG) "$(BINARY_PLATFORMS)" $(ARTIFACTS_PATH) $(GIT_HASH) $(MAKE_ROOT)/$(BINARY_DEPS_DIR) ########### DO NOT EDIT ############################# diff --git a/projects/kubernetes-sigs/image-builder/Makefile b/projects/kubernetes-sigs/image-builder/Makefile index 9feddac916..8bdc331143 100644 --- a/projects/kubernetes-sigs/image-builder/Makefile +++ b/projects/kubernetes-sigs/image-builder/Makefile @@ -67,6 +67,9 @@ HAS_RELEASE_BRANCHES=true HAS_LICENSES=false SIMPLE_CREATE_TARBALLS=false +# if running locally and do not want to try and create ec2 instance +SKIP_METAL_INSTANCE_TEST?=false + # to support a no op attribution target TARGETS_ALLOWED_WITH_NO_RELEASE_BRANCH=binaries checksums attribution @@ -131,7 +134,7 @@ BUILD_OVA_TARGETS+=build-ova-ubuntu-2004-efi fake-ubuntu-2004-ova-efi.ova upload # only running release-raw-ubuntu-2004 on presubmit here since we are just validating the instance creation and do not need to run for every OS # tinkerbell/bare metal node images -BUILD_RAW_TARGETS=$(foreach os,ubuntu-2004 ubuntu-2204 redhat-8,build-raw-$(os) $(call FAKE_IMAGE_PATH_FROM_OS_WITH_VER,raw,$(os)) upload-artifacts-raw-$(os)) release-raw-ubuntu-2004 +BUILD_RAW_TARGETS=$(foreach os,ubuntu-2004 ubuntu-2204 redhat-8,build-raw-$(os) $(call FAKE_IMAGE_PATH_FROM_OS_WITH_VER,raw,$(os)) upload-artifacts-raw-$(os)) $(if $(filter true,$(SKIP_METAL_INSTANCE_TEST)),,release-raw-ubuntu-2004) # cloud stack node images BUILD_CLOUDSTACK_TARGETS=build-cloudstack-redhat-8 $(call FAKE_IMAGE_PATH_FROM_OS_WITH_VER,cloudstack,redhat-8) upload-artifacts-cloudstack-redhat-8 @@ -216,16 +219,12 @@ export GOVC_INSECURE?=true ######################## DEPS ################################# .PHONY: setup-ami-share -setup-ami-share: - @echo -e $(call TARGET_START_LOG) - jq -n '{"ami_users": "$(DEV_ACCOUNTS)", "ami_groups": "", "snapshot_users": "$(DEV_ACCOUNTS)", "snapshot_groups": ""}' > $(PACKER_AMI_SHARE_FILE) - @echo -e $(call TARGET_END_LOG) +setup-ami-share: | $$(ENABLE_LOGGING) + @jq -n '{"ami_users": "$(DEV_ACCOUNTS)", "ami_groups": "", "snapshot_users": "$(DEV_ACCOUNTS)", "snapshot_groups": ""}' > $(PACKER_AMI_SHARE_FILE) .PHONY: setup-packer-configs-% -setup-packer-configs-%: $(GIT_PATCH_TARGET) | ensure-jq ensure-yq - @echo -e $(call TARGET_START_LOG) - build/setup_packer_configs.sh $(RELEASE_BRANCH) $(IMAGE_FORMAT) $(IMAGE_OS) $(ARTIFACTS_BUCKET) $(FINAL_IMAGE_DIR) $(ADDITIONAL_PAUSE_$(RELEASE_BRANCH)_FROM) $(LATEST) $(IMAGE_BUILDER_DIR) $(IMAGE_OS_DIR) $(IMAGE_OS_VERSION) - @echo -e $(call TARGET_END_LOG) +setup-packer-configs-%: $(GIT_PATCH_TARGET) | ensure-jq ensure-yq $$(ENABLE_LOGGING) + @build/setup_packer_configs.sh $(RELEASE_BRANCH) $(IMAGE_FORMAT) $(IMAGE_OS) $(ARTIFACTS_BUCKET) $(FINAL_IMAGE_DIR) $(ADDITIONAL_PAUSE_$(RELEASE_BRANCH)_FROM) $(LATEST) $(IMAGE_BUILDER_DIR) $(IMAGE_OS_DIR) $(IMAGE_OS_VERSION) ############################################################## @@ -233,11 +232,8 @@ setup-packer-configs-%: $(GIT_PATCH_TARGET) | ensure-jq ensure-yq # Since we do not build the ova in presubmit but want to validate upload-artifacts behavior # ex: fake-ubuntu-2004-raw.gz,fake-ubuntu-2004-ova-efi.ova fake-%: PACKER_LOG_PATH=$(ARTIFACTS_PATH)/$(subst -,/,$(basename $(*F)))/packer.log -fake-%: - @echo -e $(call TARGET_START_LOG) - touch $(@) - touch $(PACKER_LOG_PATH) - @echo -e $(call TARGET_END_LOG) +fake-%: | $$(ENABLE_LOGGING) + @touch $(@) $(PACKER_LOG_PATH) packer/ova/vsphere.json: echo "{}" > $@ @@ -268,10 +264,8 @@ $(ARTIFACTS_PATH)/bottlerocket/%: | ensure-yq ensure-tuftool # ex: upload-bottlerocket-1-ova, upload-bottlerocket-1-raw upload-bottlerocket-%: IMAGE_OS_VERSION=$(word 1,$(subst -, ,$*)) upload-bottlerocket-%: IMAGE_FORMAT=$(word 2,$(subst -, ,$*)) -upload-bottlerocket-%: $$(FINAL_IMAGE_PATH) - @echo -e $(call TARGET_START_LOG) - build/upload_bottlerocket_artifacts.sh $(RELEASE_BRANCH) $(IMAGE_FORMAT) $(ARTIFACTS_PATH_IMAGE_FORMAT) $(ARTIFACTS_UPLOAD_PATH_IMAGE_FORMAT) $(IMAGE_OS_VERSION) - @echo -e $(call TARGET_END_LOG) +upload-bottlerocket-%: $$(FINAL_IMAGE_PATH) | $$(ENABLE_LOGGING) + @build/upload_bottlerocket_artifacts.sh $(RELEASE_BRANCH) $(IMAGE_FORMAT) $(ARTIFACTS_PATH_IMAGE_FORMAT) $(ARTIFACTS_UPLOAD_PATH_IMAGE_FORMAT) $(IMAGE_OS_VERSION) ############################################################### ########################### FINAL TARGETS ############################## @@ -292,10 +286,8 @@ build-%: IMAGE_FORMAT=$(word 1,$(subst -, ,$*)) build-%: IMAGE_OS=$(word 2,$(subst -, ,$*)) build-%: IMAGE_OS_VERSION=$(word 3,$(subst -, ,$*)) build-%: IMAGE_OS_FIRMWARE=$(word 4,$(subst -, ,$*)) -build-%: validate-supported-image-% setup-packer-configs-% $(if $(filter ami,$(IMAGE_FORMAT)),setup-ami-share,) - @echo -e $(call TARGET_START_LOG) - PACKER_VAR_FILES="$(PACKER_VAR_FILES)" $(MAKE) -C $(IMAGE_BUILDER_DIR) $(call CONVERT_TO_UPSTREAM_TARGET,validate-$(IMAGE_FORMAT)-$(IMAGE_OS_WITH_VER)$(if $(IMAGE_OS_FIRMWARE),-$(IMAGE_OS_FIRMWARE),)) - @echo -e $(call TARGET_END_LOG) +build-%: validate-supported-image-% setup-packer-configs-% $(if $(filter ami,$(IMAGE_FORMAT)),setup-ami-share,) | $$(ENABLE_LOGGING) + @PACKER_VAR_FILES="$(PACKER_VAR_FILES)" $(MAKE) -C $(IMAGE_BUILDER_DIR) $(call CONVERT_TO_UPSTREAM_TARGET,validate-$(IMAGE_FORMAT)-$(IMAGE_OS_WITH_VER)$(if $(IMAGE_OS_FIRMWARE),-$(IMAGE_OS_FIRMWARE),)) ######################################################### ######################## IMAGE BUILDER TARGETS ####################### @@ -322,10 +314,8 @@ release-ami-%: validate-supported-image-% setup-ami-share | ensure-jq @echo -e $(call TARGET_END_LOG) release-ova-%: IMAGE_FORMAT=ova -release-ova-%: validate-supported-image-% | ensure-jq - @echo -e $(call TARGET_START_LOG) - build/build_image.sh $(IMAGE_OS) $(IMAGE_OS_VERSION) $(RELEASE_BRANCH) $(IMAGE_FORMAT) $(ARTIFACTS_BUCKET) $(LATEST) $(IMAGE_OS_FIRMWARE) - @echo -e $(call TARGET_END_LOG) +release-ova-%: validate-supported-image-% | ensure-jq $$(ENABLE_LOGGING) + @build/build_image.sh $(IMAGE_OS) $(IMAGE_OS_VERSION) $(RELEASE_BRANCH) $(IMAGE_FORMAT) $(ARTIFACTS_BUCKET) $(LATEST) $(IMAGE_OS_FIRMWARE) release-raw-%: IMAGE_FORMAT=raw release-raw-%: release-image-build-on-metal-% @@ -338,10 +328,8 @@ release-cloudstack-%: release-image-build-on-metal-% # used for cloudstack and raw # clone the repo first since its scp'd to the temporary instance which runs kvm release-image-build-on-metal-%: IMAGE_FORMAT?=raw -release-image-build-on-metal-%: validate-supported-image-% $(GIT_PATCH_TARGET) | ensure-jq - @echo -e $(call TARGET_START_LOG) - build/build_image_on_metal.sh $(BASE_DIRECTORY) $(PROJECT_PATH) $(RELEASE_BRANCH) $(RAW_IMAGE_BUILD_AMI) $(RAW_IMAGE_BUILD_INSTANCE_TYPE) $(RAW_IMAGE_BUILD_KEY_NAME) $(IMAGE_OS) $(IMAGE_OS_VERSION) $(IMAGE_FORMAT) $(LATEST) $(FINAL_IMAGE_DIR) $(BRANCH_NAME) - @echo -e $(call TARGET_END_LOG) +release-image-build-on-metal-%: validate-supported-image-% $(GIT_PATCH_TARGET) | ensure-jq $$(ENABLE_LOGGING) + @build/build_image_on_metal.sh $(BASE_DIRECTORY) $(PROJECT_PATH) $(RELEASE_BRANCH) $(RAW_IMAGE_BUILD_AMI) $(RAW_IMAGE_BUILD_INSTANCE_TYPE) $(RAW_IMAGE_BUILD_KEY_NAME) $(IMAGE_OS) $(IMAGE_OS_VERSION) $(IMAGE_FORMAT) $(LATEST) $(FINAL_IMAGE_DIR) $(BRANCH_NAME) ####################################################################### @@ -369,15 +357,11 @@ local-build-%: validate-supported-image-% setup-packer-configs-% ####################################################################### ######################## ARTIFACT TARGETS ############################# -s3-artifacts-%: $$(FINAL_IMAGE_PATH) - @echo -e $(call TARGET_START_LOG) - $(MAKE) -C $(MAKE_ROOT) s3-artifacts ARTIFACTS_PATH=$(ARTIFACTS_PATH_IMAGE_FORMAT) IMAGE_FORMAT=$(IMAGE_FORMAT) IMAGE_OS=$(IMAGE_OS) IMAGE_OS_VERSION=$(IMAGE_OS_VERSION) - @echo -e $(call TARGET_END_LOG) +s3-artifacts-%: $$(FINAL_IMAGE_PATH) | $$(ENABLE_LOGGING) + @$(MAKE) -C $(MAKE_ROOT) s3-artifacts ARTIFACTS_PATH=$(ARTIFACTS_PATH_IMAGE_FORMAT) IMAGE_FORMAT=$(IMAGE_FORMAT) IMAGE_OS=$(IMAGE_OS) IMAGE_OS_VERSION=$(IMAGE_OS_VERSION) -upload-artifacts-%: s3-artifacts-% - @echo -e $(call TARGET_START_LOG) - $(MAKE) -C $(MAKE_ROOT) upload-artifacts ARTIFACTS_PATH=$(ARTIFACTS_PATH_IMAGE_FORMAT) ARTIFACTS_UPLOAD_PATH=$(ARTIFACTS_UPLOAD_PATH_IMAGE_FORMAT) IMAGE_FORMAT=$(IMAGE_FORMAT) IMAGE_OS=$(IMAGE_OS) IMAGE_OS_VERSION=$(IMAGE_OS_VERSION) - @echo -e $(call TARGET_END_LOG) +upload-artifacts-%: s3-artifacts-% | $$(ENABLE_LOGGING) + @$(MAKE) -C $(MAKE_ROOT) upload-artifacts ARTIFACTS_PATH=$(ARTIFACTS_PATH_IMAGE_FORMAT) ARTIFACTS_UPLOAD_PATH=$(ARTIFACTS_UPLOAD_PATH_IMAGE_FORMAT) IMAGE_FORMAT=$(IMAGE_FORMAT) IMAGE_OS=$(IMAGE_OS) IMAGE_OS_VERSION=$(IMAGE_OS_VERSION) ####################################################################### diff --git a/projects/kubernetes-sigs/kind/Makefile b/projects/kubernetes-sigs/kind/Makefile index 123222c5a2..dd61e032d7 100644 --- a/projects/kubernetes-sigs/kind/Makefile +++ b/projects/kubernetes-sigs/kind/Makefile @@ -124,15 +124,11 @@ kind-node/images/push: LATEST_TAG=$(NODE_IMAGE_LATEST_TAG) kind-node/images/push: IMAGE_TARGET=node kind-node/images/push: $(KIND_NODE_BUILD_AMD64_TARGET) $(KIND_NODE_BUILD_ARM64_TARGET) $(ARM_ENV_CONF_TARGET) -$(KIND_BASE_IMAGE_BUILD_ARGS): | ensure-yq - @echo -e $(call TARGET_START_LOG) - build/base-image-build-args.sh $(RELEASE_BRANCH) $(ARTIFACTS_BUCKET) $@ $(LATEST) - @echo -e $(call TARGET_END_LOG) - -$(KIND_NODE_IMAGE_BUILD_ARGS): | ensure-yq - @echo -e $(call TARGET_START_LOG) - build/node-image-build-args.sh $(RELEASE_BRANCH) $(KINDNETD_IMAGE_COMPONENT) $(IMAGE_REPO) "$(ARTIFACTS_BUCKET)" $(IMAGE_TAG) $(LATEST) $@ - @echo -e $(call TARGET_END_LOG) +$(KIND_BASE_IMAGE_BUILD_ARGS): | ensure-yq $$(ENABLE_LOGGING) + @build/base-image-build-args.sh $(RELEASE_BRANCH) $(ARTIFACTS_BUCKET) $@ $(LATEST) + +$(KIND_NODE_IMAGE_BUILD_ARGS): | ensure-yq $$(ENABLE_LOGGING) + @build/node-image-build-args.sh $(RELEASE_BRANCH) $(KINDNETD_IMAGE_COMPONENT) $(IMAGE_REPO) "$(ARTIFACTS_BUCKET)" $(IMAGE_TAG) $(LATEST) $@ # Tweak the kind/base image to have a hardcode kubeadm config # so that during the image pull phase it pulls eks-d images