From 080912a7214325375963a572c01c64bf83d3019b Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 31 Jul 2024 16:47:50 -0700 Subject: [PATCH] CI: use generated identity w/ storage privileges Signed-off-by: Jack Francis --- hack/log/redact.sh | 8 -------- scripts/kind-with-registry.sh | 12 +++++++++++- .../cluster-template-prow-ci-version-dual-stack.yaml | 2 +- .../ci/cluster-template-prow-ci-version-ipv6.yaml | 2 +- .../test/ci/cluster-template-prow-ci-version.yaml | 4 ++-- .../machine-deployment-ci-version-control-plane.yaml | 2 +- .../machine-deployment-ci-version-windows.yaml | 2 +- .../test/dev/cluster-template-custom-builds.yaml | 6 +++--- .../machine-deployment-pr-version-windows.yaml | 2 +- .../patches/machine-deployment-pr-version.yaml | 4 ++-- .../data/kubetest/upstream-windows-ginkgo-v2.yaml | 2 +- .../upstream-windows-serial-slow-ginkgo-v2.yaml | 2 +- .../data/kubetest/upstream-windows-serial-slow.yaml | 2 +- test/e2e/data/kubetest/upstream-windows.yaml | 2 +- 14 files changed, 27 insertions(+), 25 deletions(-) diff --git a/hack/log/redact.sh b/hack/log/redact.sh index 4fb2ec4c389..efa6d2a6aad 100755 --- a/hack/log/redact.sh +++ b/hack/log/redact.sh @@ -24,16 +24,8 @@ log_files=() while IFS='' read -r line; do log_files+=("$line"); done < <(find "${ARTIFACTS:-${PWD}/_artifacts}" -type f) redact_vars=( "${AZURE_CLIENT_ID:-}" - "${AZURE_CLIENT_SECRET:-}" - "${AZURE_SUBSCRIPTION_ID:-}" - "${AZURE_TENANT_ID:-}" "${AZURE_JSON_B64:-}" - "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY:-}" - "$(echo -n "${AZURE_SUBSCRIPTION_ID:-}" | base64 | tr -d '\n')" - "$(echo -n "${AZURE_TENANT_ID:-}" | base64 | tr -d '\n')" - "$(echo -n "${AZURE_CLIENT_ID:-}" | base64 | tr -d '\n')" "$(echo -n "${AZURE_CLIENT_SECRET:-}" | base64 | tr -d '\n')" - "$(echo -n "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY:-}" | base64 | tr -d '\n')" ) for log_file in "${log_files[@]}"; do diff --git a/scripts/kind-with-registry.sh b/scripts/kind-with-registry.sh index 3c2830842cd..4c1a427b50a 100755 --- a/scripts/kind-with-registry.sh +++ b/scripts/kind-with-registry.sh @@ -106,6 +106,10 @@ function checkAZWIENVPreReqsAndCreateFiles() { if ! az storage account show --name "${AZWI_STORAGE_ACCOUNT}" --resource-group "${AZWI_RESOURCE_GROUP}" > /dev/null 2>&1; then echo "Creating storage account '${AZWI_STORAGE_ACCOUNT}' in '${AZWI_RESOURCE_GROUP}'" az storage account create --resource-group "${AZWI_RESOURCE_GROUP}" --name "${AZWI_STORAGE_ACCOUNT}" --output none --only-show-errors --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" + until az storage account show --name "${AZWI_STORAGE_ACCOUNT}" --resource-group "${AZWI_RESOURCE_GROUP}" > /dev/null 2>&1; do + sleep 5 + done + echo "Configuring storage account '${AZWI_STORAGE_ACCOUNT}' as static website" az storage blob service-properties update --account-name "${AZWI_STORAGE_ACCOUNT}" --static-website fi @@ -155,7 +159,13 @@ EOF AZURE_IDENTITY_ID_PRINCIPAL_ID=$(az identity show -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" --query principalId -o tsv) echo "${AZURE_IDENTITY_ID}" > "${AZURE_IDENTITY_ID_FILEPATH}" - until az role assignment create --assignee-object-id "${AZURE_IDENTITY_ID_PRINCIPAL_ID}" --role "Owner" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal --output none --only-show-errors; do + until az role assignment create --assignee-object-id "${AZURE_IDENTITY_ID_PRINCIPAL_ID}" --role "Owner" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal; do + sleep 5 + done + until az role assignment create --assignee-object-id "${AZURE_IDENTITY_ID_PRINCIPAL_ID}" --role "Storage Account Contributor" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal; do + sleep 5 + done + until az role assignment create --assignee-object-id "${AZURE_IDENTITY_ID_PRINCIPAL_ID}" --role "Storage Blob Data Owner" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal; do sleep 5 done diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 0dba073b64b..6e5992de305 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -258,7 +258,7 @@ spec: osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- apiVersion: cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index 07c551c1614..66047b73c30 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -265,7 +265,7 @@ spec: osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- apiVersion: cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index a3f378d9804..9b96145ee66 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -236,7 +236,7 @@ spec: osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -457,7 +457,7 @@ spec: osType: Windows sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-control-plane.yaml b/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-control-plane.yaml index 239f162a9e0..8d684ecd158 100644 --- a/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-control-plane.yaml +++ b/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-control-plane.yaml @@ -22,5 +22,5 @@ spec: osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} diff --git a/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-windows.yaml b/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-windows.yaml index f4ba9867cbf..a0e8936019e 100644 --- a/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-windows.yaml +++ b/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-windows.yaml @@ -15,4 +15,4 @@ spec: version: "latest" identity: UserAssigned userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 68dad17ebf6..b7242105443 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -228,7 +228,7 @@ spec: osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -277,7 +277,7 @@ spec: osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} vmExtensions: - name: CustomScript protectedSettings: @@ -406,7 +406,7 @@ spec: osType: Windows sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml index c20e2859a3e..c153570d2eb 100644 --- a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml +++ b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml @@ -8,7 +8,7 @@ spec: spec: identity: UserAssigned userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} image: # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. # latest binaries and images will get replaced to the desired version by the script above. diff --git a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml index faf7816c7f0..60e367449c6 100644 --- a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml +++ b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml @@ -7,7 +7,7 @@ spec: spec: identity: UserAssigned userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} image: # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. # latest binaries and images will get replaced to the desired version by the script above. @@ -26,7 +26,7 @@ spec: spec: identity: UserAssigned userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} image: # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. # latest binaries and images will get replaced to the desired version by the script above. diff --git a/test/e2e/data/kubetest/upstream-windows-ginkgo-v2.yaml b/test/e2e/data/kubetest/upstream-windows-ginkgo-v2.yaml index e6e6b6c5a86..3b5b381238f 100644 --- a/test/e2e/data/kubetest/upstream-windows-ginkgo-v2.yaml +++ b/test/e2e/data/kubetest/upstream-windows-ginkgo-v2.yaml @@ -1,5 +1,5 @@ ginkgo.focus: \[Conformance\]|\[NodeConformance\]|\[sig-windows\]|\[sig-apps\].CronJob|\[sig-api-machinery\].ResourceQuota|\[sig-scheduling\].SchedulerPreemption -ginkgo.skip: \[LinuxOnly\]|\[Serial\]|\[Slow\]|\[Excluded:WindowsDocker\]|Networking.Granular.Checks(.*)node-pod.communication|Guestbook.application.should.create.and.stop.a.working.application|device.plugin.for.Windows|Container.Lifecycle.Hook.when.create.a.pod.with.lifecycle.hook.should.execute(.*)http.hook.properly|\[sig-api-machinery\].Garbage.collector +ginkgo.skip: \[LinuxOnly\]|\[Serial\]|\[Slow\]|\[Excluded:WindowsDocker\]|\[Feature:DynamicResourceAllocation\]|Networking.Granular.Checks(.*)node-pod.communication|Guestbook.application.should.create.and.stop.a.working.application|device.plugin.for.Windows|Container.Lifecycle.Hook.when.create.a.pod.with.lifecycle.hook.should.execute(.*)http.hook.properly|\[sig-api-machinery\].Garbage.collector disable-log-dump: true ginkgo.progress: true ginkgo.slow-spec-threshold: 120s diff --git a/test/e2e/data/kubetest/upstream-windows-serial-slow-ginkgo-v2.yaml b/test/e2e/data/kubetest/upstream-windows-serial-slow-ginkgo-v2.yaml index 3798dffbaf2..17a1578ffab 100644 --- a/test/e2e/data/kubetest/upstream-windows-serial-slow-ginkgo-v2.yaml +++ b/test/e2e/data/kubetest/upstream-windows-serial-slow-ginkgo-v2.yaml @@ -1,5 +1,5 @@ ginkgo.focus: (\[sig-windows\]|\[sig-scheduling\].SchedulerPreemption|\[sig-autoscaling\].\[Feature:HPA\]|\[sig-apps\].CronJob).*(\[Serial\]|\[Slow\])|(\[Serial\]|\[Slow\]).*(\[Conformance\]|\[NodeConformance\])|\[sig-api-machinery\].Garbage.collector -ginkgo.skip: \[LinuxOnly\]|\[Excluded:WindowsDocker\]|device.plugin.for.Windows +ginkgo.skip: \[LinuxOnly\]|\[Excluded:WindowsDocker\]|\[Feature:DynamicResourceAllocation\]|device.plugin.for.Windows disable-log-dump: true ginkgo.progress: true ginkgo.slow-spec-threshold: 120s diff --git a/test/e2e/data/kubetest/upstream-windows-serial-slow.yaml b/test/e2e/data/kubetest/upstream-windows-serial-slow.yaml index ce324e24e2e..0790af8232b 100644 --- a/test/e2e/data/kubetest/upstream-windows-serial-slow.yaml +++ b/test/e2e/data/kubetest/upstream-windows-serial-slow.yaml @@ -1,5 +1,5 @@ ginkgo.focus: (\[sig-windows\]|\[sig-scheduling\].SchedulerPreemption|\[sig-autoscaling\].\[Feature:HPA\]|\[sig-apps\].CronJob).*(\[Serial\]|\[Slow\])|(\[Serial\]|\[Slow\]).*(\[Conformance\]|\[NodeConformance\])|\[sig-api-machinery\].Garbage.collector -ginkgo.skip: \[LinuxOnly\]|\[Excluded:WindowsDocker\]|device.plugin.for.Windows +ginkgo.skip: \[LinuxOnly\]|\[Excluded:WindowsDocker\]|\[Feature:DynamicResourceAllocation\]|device.plugin.for.Windows disable-log-dump: true ginkgo.progress: true ginkgo.slowSpecThreshold: 120.0 diff --git a/test/e2e/data/kubetest/upstream-windows.yaml b/test/e2e/data/kubetest/upstream-windows.yaml index 7d3d0a73111..eba90e9f33d 100644 --- a/test/e2e/data/kubetest/upstream-windows.yaml +++ b/test/e2e/data/kubetest/upstream-windows.yaml @@ -1,5 +1,5 @@ ginkgo.focus: \[Conformance\]|\[NodeConformance\]|\[sig-windows\]|\[sig-apps\].CronJob|\[sig-api-machinery\].ResourceQuota|\[sig-scheduling\].SchedulerPreemption -ginkgo.skip: \[LinuxOnly\]|\[Serial\]|\[Slow\]|\[Excluded:WindowsDocker\]|Networking.Granular.Checks(.*)node-pod.communication|Guestbook.application.should.create.and.stop.a.working.application|device.plugin.for.Windows|Container.Lifecycle.Hook.when.create.a.pod.with.lifecycle.hook.should.execute(.*)http.hook.properly|\[sig-api-machinery\].Garbage.collector +ginkgo.skip: \[LinuxOnly\]|\[Serial\]|\[Slow\]|\[Excluded:WindowsDocker\]|\[Feature:DynamicResourceAllocation\]|Networking.Granular.Checks(.*)node-pod.communication|Guestbook.application.should.create.and.stop.a.working.application|device.plugin.for.Windows|Container.Lifecycle.Hook.when.create.a.pod.with.lifecycle.hook.should.execute(.*)http.hook.properly|\[sig-api-machinery\].Garbage.collector disable-log-dump: true ginkgo.progress: true ginkgo.slowSpecThreshold: 120.0