From a82c2f8d4b9171f1b60e03c38dc04b750c02fd54 Mon Sep 17 00:00:00 2001 From: Shizhao Liu <43278392+2ez4szliu@users.noreply.github.com> Date: Sun, 15 Dec 2024 08:19:56 -0800 Subject: [PATCH] [release-0.20] Remove unnecessary builds from batch buildspecs (#4135) (#4137) Co-authored-by: Abhay Krishna --- Common.mk | 2 +- build/lib/validate_release_buildspecs.sh | 22 +++-- .../kubernetes-sigs/image-builder/Makefile | 2 +- .../kubernetes/cloud-provider-aws/Makefile | 1 + release/checksums-build.yml | 36 -------- release/staging-build.yml | 87 ------------------- tools/version-tracker/buildspecs/upgrade.yml | 32 ------- 7 files changed, 19 insertions(+), 163 deletions(-) diff --git a/Common.mk b/Common.mk index 6bce26f6e1..f5086ad319 100644 --- a/Common.mk +++ b/Common.mk @@ -98,7 +98,7 @@ REPO_SPARSE_CHECKOUT?= #################### RELEASE BRANCHES ############## HAS_RELEASE_BRANCHES?=false RELEASE_BRANCH?= -SUPPORTED_K8S_VERSIONS=$(shell cat $(BASE_DIRECTORY)/release/SUPPORTED_RELEASE_BRANCHES) +SUPPORTED_K8S_VERSIONS?=$(shell cat $(BASE_DIRECTORY)/release/SUPPORTED_RELEASE_BRANCHES) # Comma-separated list of Kubernetes versions to skip building artifacts for SKIPPED_K8S_VERSIONS?= BINARIES_ARE_RELEASE_BRANCHED?=true diff --git a/build/lib/validate_release_buildspecs.sh b/build/lib/validate_release_buildspecs.sh index 2f1c2c45c8..5ffe4e430f 100755 --- a/build/lib/validate_release_buildspecs.sh +++ b/build/lib/validate_release_buildspecs.sh @@ -19,15 +19,23 @@ set -o nounset set -o pipefail REPO_ROOT="$(git rev-parse --show-toplevel)" -RELEASE_BUILDSPECS=("$REPO_ROOT/release/checksums-build.yml" "$REPO_ROOT/release/staging-build.yml") +RELEASE_BUILDSPECS=("$REPO_ROOT/release/checksums-build.yml" "$REPO_ROOT/release/staging-build.yml" "$REPO_ROOT/tools/version-tracker/buildspecs/upgrade.yml") VALIDATIONS_FAILED=0 for buildspec in "${RELEASE_BUILDSPECS[@]}"; do + echo "Validating builds count in build graph for buildspec - $buildspec" + num_builds_in_batch=$(yq ".batch.build-graph | length" $buildspec) + if [[ $num_builds_in_batch -ge 100 ]]; then + echo "Maximum allowed builds in batch is 100, current number of builds: $num_builds_in_batch" + VALIDATIONS_FAILED=1 + INVALID_BUILDSPEC="true" + fi + depends_on_list=($(yq "[.batch.build-graph[].depend-on[] | select(. != \"null\")] | unique | .[]" $buildspec)) identifier_list=($(yq ".batch.build-graph[].identifier" $buildspec)) INVALID_BUILDSPEC="false" - echo "Validating identifier unqiueness in the buildspec - $buildspec" + echo "Validating identifier uniqueness in the buildspec - $buildspec" duplicate_ids=($(printf '%s\n' "${identifier_list[@]}"|awk '!($0 in seen){seen[$0];next} 1' | uniq)) if [ "${#duplicate_ids[@]}" -gt 0 ]; then printf -v duplicate_id_csv '%s,' "${duplicate_ids[@]}" @@ -36,10 +44,12 @@ for buildspec in "${RELEASE_BUILDSPECS[@]}"; do INVALID_BUILDSPEC="true" fi - echo "Validating identifiers in depend-on list are valid identifiers in build graph in the buildspec - $buildspec" - invalid_dependencies=($(for dependency in ${depends_on_list[@]}; do - [[ ${identifier_list[*]} =~ (^|[[:space:]])"$dependency"($|[[:space:]]) ]] || echo "$dependency" - done)) + if [ "${#depends_on_list[@]}" -gt 0 ]; then + echo "Validating identifiers in depend-on list are valid identifiers in build graph in the buildspec - $buildspec" + invalid_dependencies=($(for dependency in ${depends_on_list[@]}; do + [[ ${identifier_list[*]} =~ (^|[[:space:]])"$dependency"($|[[:space:]]) ]] || echo "$dependency" + done)) + fi if [ "${#invalid_dependencies[@]}" -gt 0 ]; then printf -v invalid_deps_csv '%s,' "${invalid_dependencies[@]}" diff --git a/projects/kubernetes-sigs/image-builder/Makefile b/projects/kubernetes-sigs/image-builder/Makefile index c858d0f635..41ca239f8b 100644 --- a/projects/kubernetes-sigs/image-builder/Makefile +++ b/projects/kubernetes-sigs/image-builder/Makefile @@ -204,7 +204,7 @@ endif ##################################################################### -BUILDSPECS=projects/$(COMPONENT)/buildspecs/ova.yml projects/$(COMPONENT)/buildspecs/raw.yml projects/$(COMPONENT)/buildspecs/ami.yml +BUILDSPECS=projects/$(COMPONENT)/buildspecs/ova.yml projects/$(COMPONENT)/buildspecs/raw.yml BUILDSPEC_DEPENDS_ON_OVERRIDE=none BUILDSPEC_VARS_KEYS=IMAGE_OS RELEASE_BRANCH BUILDSPEC_VARS_VALUES=SUPPORTED_OS_DISTROS SUPPORTED_K8S_VERSIONS diff --git a/projects/kubernetes/cloud-provider-aws/Makefile b/projects/kubernetes/cloud-provider-aws/Makefile index 4fe1893d5b..64fafe9f72 100644 --- a/projects/kubernetes/cloud-provider-aws/Makefile +++ b/projects/kubernetes/cloud-provider-aws/Makefile @@ -12,6 +12,7 @@ EXTRA_GO_LDFLAGS=-X k8s.io/component-base/version.gitVersion=$(shell cat ./$(REL HAS_S3_ARTIFACTS=true HAS_RELEASE_BRANCHES=true BUILDSPEC_VARS_KEYS=RELEASE_BRANCH +SUPPORTED_K8S_VERSIONS=1-27 BUILDSPEC_VARS_VALUES=SUPPORTED_K8S_VERSIONS IMAGE_NAMES= diff --git a/release/checksums-build.yml b/release/checksums-build.yml index 78a79278d4..69d3fd8981 100644 --- a/release/checksums-build.yml +++ b/release/checksums-build.yml @@ -244,14 +244,6 @@ batch: PROJECT_PATH: projects/kubernetes/autoscaler CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.autoscaler RELEASE_BRANCH: 1-30 - - identifier: kubernetes_cloud_provider_aws_1_26 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-26 - identifier: kubernetes_cloud_provider_aws_1_27 env: type: ARM_CONTAINER @@ -260,30 +252,6 @@ batch: PROJECT_PATH: projects/kubernetes/cloud-provider-aws CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws RELEASE_BRANCH: 1-27 - - identifier: kubernetes_cloud_provider_aws_1_28 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-28 - - identifier: kubernetes_cloud_provider_aws_1_29 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-29 - - identifier: kubernetes_cloud_provider_aws_1_30 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-30 - identifier: kubernetes_cloud_provider_vsphere_1_26 env: type: ARM_CONTAINER @@ -531,11 +499,7 @@ batch: - kubernetes_autoscaler_1_28 - kubernetes_autoscaler_1_29 - kubernetes_autoscaler_1_30 - - kubernetes_cloud_provider_aws_1_26 - kubernetes_cloud_provider_aws_1_27 - - kubernetes_cloud_provider_aws_1_28 - - kubernetes_cloud_provider_aws_1_29 - - kubernetes_cloud_provider_aws_1_30 - kubernetes_cloud_provider_vsphere_1_26 - kubernetes_cloud_provider_vsphere_1_27 - kubernetes_cloud_provider_vsphere_1_28 diff --git a/release/staging-build.yml b/release/staging-build.yml index b7f1a94173..ad922890ef 100644 --- a/release/staging-build.yml +++ b/release/staging-build.yml @@ -283,14 +283,6 @@ batch: variables: PROJECT_PATH: projects/kube-vip/kube-vip CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kube-vip.kube-vip - - identifier: kubernetes_cloud_provider_aws_1_26 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-26 - identifier: kubernetes_cloud_provider_aws_1_27 env: type: ARM_CONTAINER @@ -299,30 +291,6 @@ batch: PROJECT_PATH: projects/kubernetes/cloud-provider-aws CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws RELEASE_BRANCH: 1-27 - - identifier: kubernetes_cloud_provider_aws_1_28 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-28 - - identifier: kubernetes_cloud_provider_aws_1_29 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-29 - - identifier: kubernetes_cloud_provider_aws_1_30 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-30 - identifier: kubernetes_cloud_provider_vsphere_1_26 env: type: ARM_CONTAINER @@ -508,61 +476,6 @@ batch: IMAGE_OS: bottlerocket RELEASE_BRANCH: 1-30 IMAGE_OS_VERSION: "1" - - identifier: kubernetes_sigs_image_builder_bottlerocket_1_26_ami - buildspec: projects/kubernetes-sigs/image-builder/buildspecs/ami.yml - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes-sigs/image-builder - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes-sigs.image-builder - IMAGE_OS: bottlerocket - RELEASE_BRANCH: 1-26 - IMAGE_OS_VERSION: "1" - - identifier: kubernetes_sigs_image_builder_bottlerocket_1_27_ami - buildspec: projects/kubernetes-sigs/image-builder/buildspecs/ami.yml - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes-sigs/image-builder - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes-sigs.image-builder - IMAGE_OS: bottlerocket - RELEASE_BRANCH: 1-27 - IMAGE_OS_VERSION: "1" - - identifier: kubernetes_sigs_image_builder_bottlerocket_1_28_ami - buildspec: projects/kubernetes-sigs/image-builder/buildspecs/ami.yml - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes-sigs/image-builder - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes-sigs.image-builder - IMAGE_OS: bottlerocket - RELEASE_BRANCH: 1-28 - IMAGE_OS_VERSION: "1" - - identifier: kubernetes_sigs_image_builder_bottlerocket_1_29_ami - buildspec: projects/kubernetes-sigs/image-builder/buildspecs/ami.yml - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes-sigs/image-builder - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes-sigs.image-builder - IMAGE_OS: bottlerocket - RELEASE_BRANCH: 1-29 - IMAGE_OS_VERSION: "1" - - identifier: kubernetes_sigs_image_builder_bottlerocket_1_30_ami - buildspec: projects/kubernetes-sigs/image-builder/buildspecs/ami.yml - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes-sigs/image-builder - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes-sigs.image-builder - IMAGE_OS: bottlerocket - RELEASE_BRANCH: 1-30 - IMAGE_OS_VERSION: "1" - identifier: kubernetes_sigs_kind buildspec: projects/kubernetes-sigs/kind/buildspecs/binaries.yml env: diff --git a/tools/version-tracker/buildspecs/upgrade.yml b/tools/version-tracker/buildspecs/upgrade.yml index 05127591ff..ef666f66ae 100644 --- a/tools/version-tracker/buildspecs/upgrade.yml +++ b/tools/version-tracker/buildspecs/upgrade.yml @@ -236,14 +236,6 @@ batch: PROJECT_PATH: projects/kubernetes/autoscaler CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.autoscaler RELEASE_BRANCH: 1-30 - - identifier: kubernetes_cloud_provider_aws_1_26 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-26 - identifier: kubernetes_cloud_provider_aws_1_27 env: type: ARM_CONTAINER @@ -252,30 +244,6 @@ batch: PROJECT_PATH: projects/kubernetes/cloud-provider-aws CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws RELEASE_BRANCH: 1-27 - - identifier: kubernetes_cloud_provider_aws_1_28 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-28 - - identifier: kubernetes_cloud_provider_aws_1_29 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-29 - - identifier: kubernetes_cloud_provider_aws_1_30 - env: - type: ARM_CONTAINER - compute-type: BUILD_GENERAL1_SMALL - variables: - PROJECT_PATH: projects/kubernetes/cloud-provider-aws - CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws - RELEASE_BRANCH: 1-30 - identifier: kubernetes_cloud_provider_vsphere_1_26 env: type: ARM_CONTAINER