Merge pull request #5043 from jackfrancis/custom-builds-use-generated…

…-identity CI: use generated identity w/ storage privileges
kubernetes-sigs · Aug 2, 2024 · a91680a · a91680a
2 parents a498485 + 080912a
commit a91680a
Show file tree

Hide file tree

Showing 14 changed files with 27 additions and 25 deletions.
diff --git a/hack/log/redact.sh b/hack/log/redact.sh
@@ -24,16 +24,8 @@ log_files=()
 while IFS='' read -r line; do log_files+=("$line"); done < <(find "${ARTIFACTS:-${PWD}/_artifacts}" -type f)
 redact_vars=(
     "${AZURE_CLIENT_ID:-}"
-    "${AZURE_CLIENT_SECRET:-}"
-    "${AZURE_SUBSCRIPTION_ID:-}"
-    "${AZURE_TENANT_ID:-}"
     "${AZURE_JSON_B64:-}"
-    "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY:-}"
-    "$(echo -n "${AZURE_SUBSCRIPTION_ID:-}" | base64 | tr -d '\n')"
-    "$(echo -n "${AZURE_TENANT_ID:-}" | base64 | tr -d '\n')"
-    "$(echo -n "${AZURE_CLIENT_ID:-}" | base64 | tr -d '\n')"
     "$(echo -n "${AZURE_CLIENT_SECRET:-}" | base64 | tr -d '\n')"
-    "$(echo -n "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY:-}" | base64 | tr -d '\n')"
 )
 
 for log_file in "${log_files[@]}"; do

diff --git a/scripts/kind-with-registry.sh b/scripts/kind-with-registry.sh
@@ -106,6 +106,10 @@ function checkAZWIENVPreReqsAndCreateFiles() {
     if ! az storage account show --name "${AZWI_STORAGE_ACCOUNT}" --resource-group "${AZWI_RESOURCE_GROUP}" > /dev/null 2>&1; then
       echo "Creating storage account '${AZWI_STORAGE_ACCOUNT}' in '${AZWI_RESOURCE_GROUP}'"
       az storage account create --resource-group "${AZWI_RESOURCE_GROUP}" --name "${AZWI_STORAGE_ACCOUNT}" --output none --only-show-errors --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}"
+      until az storage account show --name "${AZWI_STORAGE_ACCOUNT}" --resource-group "${AZWI_RESOURCE_GROUP}" > /dev/null 2>&1; do
+        sleep 5
+      done
+      echo "Configuring storage account '${AZWI_STORAGE_ACCOUNT}' as static website"
       az storage blob service-properties update --account-name "${AZWI_STORAGE_ACCOUNT}" --static-website
     fi
 
@@ -155,7 +159,13 @@ EOF
     AZURE_IDENTITY_ID_PRINCIPAL_ID=$(az identity show -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" --query principalId -o tsv)
 
     echo "${AZURE_IDENTITY_ID}" > "${AZURE_IDENTITY_ID_FILEPATH}"
-    until az role assignment create --assignee-object-id "${AZURE_IDENTITY_ID_PRINCIPAL_ID}" --role "Owner" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal --output none --only-show-errors; do
+    until az role assignment create --assignee-object-id "${AZURE_IDENTITY_ID_PRINCIPAL_ID}" --role "Owner" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal; do
+      sleep 5
+    done
+    until az role assignment create --assignee-object-id "${AZURE_IDENTITY_ID_PRINCIPAL_ID}" --role "Storage Account Contributor" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal; do
+      sleep 5
+    done
+    until az role assignment create --assignee-object-id "${AZURE_IDENTITY_ID_PRINCIPAL_ID}" --role "Storage Blob Data Owner" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal; do
       sleep 5
     done
 

diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml
diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml
diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml
diff --git a/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-control-plane.yaml b/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-control-plane.yaml
@@ -22,5 +22,5 @@ spec:
         osType: Linux
       sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
       userAssignedIdentities:
-      - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity
+      - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
       vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
diff --git a/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-windows.yaml b/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version-windows.yaml
@@ -15,4 +15,4 @@ spec:
           version: "latest"
       identity: UserAssigned
       userAssignedIdentities:
-      - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity
+      - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml
diff --git a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml
@@ -8,7 +8,7 @@ spec:
     spec:
       identity: UserAssigned
       userAssignedIdentities:
-        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity
+        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
       image:
         # we use the latest image as a workaround there is no published marketplace image for k8s CI versions.
         # latest binaries and images will get replaced to the desired version by the script above.

diff --git a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml
@@ -7,7 +7,7 @@ spec:
     spec:
       identity: UserAssigned
       userAssignedIdentities:
-        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity
+        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
       image:
         # we use the latest image as a workaround there is no published marketplace image for k8s CI versions.
         # latest binaries and images will get replaced to the desired version by the script above.
@@ -26,7 +26,7 @@ spec:
     spec:
       identity: UserAssigned
       userAssignedIdentities:
-        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/capz-ci/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity
+        - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY}
       image:
         # we use the latest image as a workaround there is no published marketplace image for k8s CI versions.
         # latest binaries and images will get replaced to the desired version by the script above.

diff --git a/test/e2e/data/kubetest/upstream-windows-ginkgo-v2.yaml b/test/e2e/data/kubetest/upstream-windows-ginkgo-v2.yaml
@@ -1,5 +1,5 @@
 ginkgo.focus: \[Conformance\]|\[NodeConformance\]|\[sig-windows\]|\[sig-apps\].CronJob|\[sig-api-machinery\].ResourceQuota|\[sig-scheduling\].SchedulerPreemption
-ginkgo.skip: \[LinuxOnly\]|\[Serial\]|\[Slow\]|\[Excluded:WindowsDocker\]|Networking.Granular.Checks(.*)node-pod.communication|Guestbook.application.should.create.and.stop.a.working.application|device.plugin.for.Windows|Container.Lifecycle.Hook.when.create.a.pod.with.lifecycle.hook.should.execute(.*)http.hook.properly|\[sig-api-machinery\].Garbage.collector
+ginkgo.skip: \[LinuxOnly\]|\[Serial\]|\[Slow\]|\[Excluded:WindowsDocker\]|\[Feature:DynamicResourceAllocation\]|Networking.Granular.Checks(.*)node-pod.communication|Guestbook.application.should.create.and.stop.a.working.application|device.plugin.for.Windows|Container.Lifecycle.Hook.when.create.a.pod.with.lifecycle.hook.should.execute(.*)http.hook.properly|\[sig-api-machinery\].Garbage.collector
 disable-log-dump: true
 ginkgo.progress: true
 ginkgo.slow-spec-threshold: 120s

diff --git a/test/e2e/data/kubetest/upstream-windows-serial-slow-ginkgo-v2.yaml b/test/e2e/data/kubetest/upstream-windows-serial-slow-ginkgo-v2.yaml
@@ -1,5 +1,5 @@
 ginkgo.focus: (\[sig-windows\]|\[sig-scheduling\].SchedulerPreemption|\[sig-autoscaling\].\[Feature:HPA\]|\[sig-apps\].CronJob).*(\[Serial\]|\[Slow\])|(\[Serial\]|\[Slow\]).*(\[Conformance\]|\[NodeConformance\])|\[sig-api-machinery\].Garbage.collector
-ginkgo.skip: \[LinuxOnly\]|\[Excluded:WindowsDocker\]|device.plugin.for.Windows
+ginkgo.skip: \[LinuxOnly\]|\[Excluded:WindowsDocker\]|\[Feature:DynamicResourceAllocation\]|device.plugin.for.Windows
 disable-log-dump: true
 ginkgo.progress: true
 ginkgo.slow-spec-threshold: 120s

diff --git a/test/e2e/data/kubetest/upstream-windows-serial-slow.yaml b/test/e2e/data/kubetest/upstream-windows-serial-slow.yaml
@@ -1,5 +1,5 @@
 ginkgo.focus: (\[sig-windows\]|\[sig-scheduling\].SchedulerPreemption|\[sig-autoscaling\].\[Feature:HPA\]|\[sig-apps\].CronJob).*(\[Serial\]|\[Slow\])|(\[Serial\]|\[Slow\]).*(\[Conformance\]|\[NodeConformance\])|\[sig-api-machinery\].Garbage.collector
-ginkgo.skip: \[LinuxOnly\]|\[Excluded:WindowsDocker\]|device.plugin.for.Windows
+ginkgo.skip: \[LinuxOnly\]|\[Excluded:WindowsDocker\]|\[Feature:DynamicResourceAllocation\]|device.plugin.for.Windows
 disable-log-dump: true
 ginkgo.progress: true
 ginkgo.slowSpecThreshold: 120.0

diff --git a/test/e2e/data/kubetest/upstream-windows.yaml b/test/e2e/data/kubetest/upstream-windows.yaml
@@ -1,5 +1,5 @@
 ginkgo.focus: \[Conformance\]|\[NodeConformance\]|\[sig-windows\]|\[sig-apps\].CronJob|\[sig-api-machinery\].ResourceQuota|\[sig-scheduling\].SchedulerPreemption
-ginkgo.skip: \[LinuxOnly\]|\[Serial\]|\[Slow\]|\[Excluded:WindowsDocker\]|Networking.Granular.Checks(.*)node-pod.communication|Guestbook.application.should.create.and.stop.a.working.application|device.plugin.for.Windows|Container.Lifecycle.Hook.when.create.a.pod.with.lifecycle.hook.should.execute(.*)http.hook.properly|\[sig-api-machinery\].Garbage.collector
+ginkgo.skip: \[LinuxOnly\]|\[Serial\]|\[Slow\]|\[Excluded:WindowsDocker\]|\[Feature:DynamicResourceAllocation\]|Networking.Granular.Checks(.*)node-pod.communication|Guestbook.application.should.create.and.stop.a.working.application|device.plugin.for.Windows|Container.Lifecycle.Hook.when.create.a.pod.with.lifecycle.hook.should.execute(.*)http.hook.properly|\[sig-api-machinery\].Garbage.collector
 disable-log-dump: true
 ginkgo.progress: true
 ginkgo.slowSpecThreshold: 120.0