feat: set workload image pull secrets (#7891)

apecloud · Aug 1, 2024 · 0bbc36b · 0bbc36b
1 parent 8a02436
commit 0bbc36b
Show file tree

Hide file tree

Showing 15 changed files with 101 additions and 4 deletions.
diff --git a/cmd/dataprotection/main.go b/cmd/dataprotection/main.go
@@ -446,5 +446,13 @@ func validateRequiredToParseConfigs() error {
 	if err := validateWorkerServiceAccountAnnotations(viper.GetString(dptypes.CfgKeyWorkerServiceAccountAnnotations)); err != nil {
 		return err
 	}
+
+	if imagePullSecrets := viper.GetString(constant.KBImagePullSecrets); imagePullSecrets != "" {
+		secrets := make([]corev1.LocalObjectReference, 0)
+		if err := json.Unmarshal([]byte(imagePullSecrets), &secrets); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
diff --git a/cmd/manager/main.go b/cmd/manager/main.go
@@ -254,6 +254,14 @@ func validateRequiredToParseConfigs() error {
 	if err := validateAffinity(viper.GetString(constant.CfgKeyDataPlaneAffinity)); err != nil {
 		return err
 	}
+
+	if imagePullSecrets := viper.GetString(constant.KBImagePullSecrets); imagePullSecrets != "" {
+		secrets := make([]corev1.LocalObjectReference, 0)
+		if err := json.Unmarshal([]byte(imagePullSecrets), &secrets); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

diff --git a/controllers/apps/operations/custom/action_exec.go b/controllers/apps/operations/custom/action_exec.go
@@ -171,6 +171,7 @@ func (e *ExecAction) buildExecPodSpec(actionCtx ActionContext,
 	return &corev1.PodSpec{
 		Containers: []corev1.Container{*container},
 		// tolerate all taints
-		Tolerations: e.Comp.Tolerations,
+		Tolerations:      e.Comp.Tolerations,
+		ImagePullSecrets: intctrlutil.BuildImagePullSecrets(),
 	}, nil
 }
diff --git a/controllers/apps/operations/datascript.go b/controllers/apps/operations/datascript.go
@@ -328,6 +328,7 @@ func buildDataScriptJobs(reqCtx intctrlutil.RequestCtx, cli client.Client, clust
 		job.Spec.BackoffLimit = pointer.Int32(0)
 		job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever
 		job.Spec.Template.Spec.Containers = []corev1.Container{container}
+		job.Spec.Template.Spec.ImagePullSecrets = intctrlutil.BuildImagePullSecrets()
 
 		// add labels
 		job.Labels = getDataScriptJobLabels(cluster.Name, component.Name, ops.Name)

diff --git a/controllers/apps/operations/rebuild_instance_inplace.go b/controllers/apps/operations/rebuild_instance_inplace.go
@@ -310,7 +310,8 @@ func (inPlaceHelper *inplaceRebuildHelper) createTmpPVCsAndPod(reqCtx intctrluti
 		AddLabels(constant.OpsRequestNameLabelKey, opsRequest.Name).
 		AddLabels(constant.OpsRequestNamespaceLabelKey, opsRequest.Namespace).
 		SetTopologySpreadConstraints(inPlaceHelper.targetPod.Spec.TopologySpreadConstraints).
-		SetAffinity(inPlaceHelper.targetPod.Spec.Affinity)
+		SetAffinity(inPlaceHelper.targetPod.Spec.Affinity).
+		SetImagePullSecrets(intctrlutil.BuildImagePullSecrets())
 	if inPlaceHelper.instance.TargetNodeName != "" {
 		rebuildPodBuilder.SetNodeSelector(map[string]string{
 			corev1.LabelHostname: inPlaceHelper.instance.TargetNodeName,

diff --git a/controllers/dataprotection/utils.go b/controllers/dataprotection/utils.go
@@ -383,6 +383,7 @@ func EnsureWorkerServiceAccount(reqCtx intctrlutil.RequestCtx, cli client.Client
 				sa.Annotations[k] = v
 			}
 		}
+		sa.ImagePullSecrets = intctrlutil.BuildImagePullSecrets()
 		if !reflect.DeepEqual(sa, saCopy) {
 			err := cli.Patch(ctx, sa, client.MergeFrom(saCopy), multicluster.InUniversalContext())
 			if err != nil {
@@ -418,6 +419,7 @@ func EnsureWorkerServiceAccount(reqCtx intctrlutil.RequestCtx, cli client.Client
 		sa.Name = saName
 		sa.Namespace = namespace
 		sa.Annotations = extraAnnotations
+		sa.ImagePullSecrets = intctrlutil.BuildImagePullSecrets()
 		if err := cli.Create(ctx, sa, multicluster.InUniversalContext()); err != nil {
 			return client.IgnoreAlreadyExists(err)
 		}

diff --git a/deploy/helm/templates/dataprotection.yaml b/deploy/helm/templates/dataprotection.yaml
@@ -96,6 +96,8 @@ spec:
             {{- end }}
             - name: KUBEBLOCKS_IMAGE_PULL_POLICY
               value: {{ .Values.dataProtection.image.pullPolicy }}
+            - name: KUBEBLOCKS_IMAGE_PULL_SECRETS
+              value: {{ .Values.dataProtection.image.imagePullSecrets | toJson | quote }}
             - name: KUBEBLOCKS_TOOLS_IMAGE
               value: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.tools.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
             - name: KUBEBLOCKS_SERVICEACCOUNT_NAME

diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml
@@ -116,6 +116,8 @@ spec:
             {{- end }}
             - name: KUBEBLOCKS_IMAGE_PULL_POLICY
               value: {{ .Values.image.pullPolicy }}
+            - name: KUBEBLOCKS_IMAGE_PULL_SECRETS
+              value: {{ .Values.image.imagePullSecrets | toJson | quote }}
             - name: KUBEBLOCKS_TOOLS_IMAGE
               value: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.tools.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
             - name: KUBEBLOCKS_DATASCRIPT_CLIENTS_IMAGE

diff --git a/deploy/helm/templates/serviceaccount.yaml b/deploy/helm/templates/serviceaccount.yaml
@@ -9,7 +9,12 @@ metadata:
   annotations:
     {{- toYaml . | nindent 4 }}
   {{- end }}
-  {{- if ( include "kubeblocks.addonControllerEnabled" . ) | deepEqual "true" }}
+  {{- with .Values.image.imagePullSecrets }}
+imagePullSecrets:
+    {{- toYaml . | nindent 2 }}
+  {{- end }}
+
+{{- if ( include "kubeblocks.addonControllerEnabled" . ) | deepEqual "true" }}
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -25,7 +30,6 @@ metadata:
 imagePullSecrets:
     {{- toYaml . | nindent 2 }}
   {{- end }}
-  {{- end }}
 {{- end }}
 
 {{- if and .Values.dataProtection.enabled }}
@@ -36,4 +40,10 @@ metadata:
   name: {{ include "dataprotection.execWorkerSAName" . }}
   labels:
     {{- include "kubeblocks.labels" . | nindent 4 }}
+  {{- with .Values.image.imagePullSecrets }}
+imagePullSecrets:
+  {{- toYaml . | nindent 2 }}
+  {{- end }}
 {{- end }}
+
+{{- end }}
diff --git a/pkg/constant/const.go b/pkg/constant/const.go
@@ -38,6 +38,7 @@ const (
 	KBServiceAccountName     = "KUBEBLOCKS_SERVICEACCOUNT_NAME"
 	KBToolsImage             = "KUBEBLOCKS_TOOLS_IMAGE"
 	KBImagePullPolicy        = "KUBEBLOCKS_IMAGE_PULL_POLICY"
+	KBImagePullSecrets       = "KUBEBLOCKS_IMAGE_PULL_SECRETS"
 	KBDataScriptClientsImage = "KUBEBLOCKS_DATASCRIPT_CLIENTS_IMAGE"
 )
 

diff --git a/pkg/controller/builder/builder_pod.go b/pkg/controller/builder/builder_pod.go
@@ -117,3 +117,8 @@ func (builder *PodBuilder) SetActiveDeadlineSeconds(activeDeadline *int64) *PodB
 	builder.get().Spec.ActiveDeadlineSeconds = activeDeadline
 	return builder
 }
+
+func (builder *PodBuilder) SetImagePullSecrets(secrets []corev1.LocalObjectReference) *PodBuilder {
+	builder.get().Spec.ImagePullSecrets = secrets
+	return builder
+}
diff --git a/pkg/controller/builder/builder_service_account.go b/pkg/controller/builder/builder_service_account.go
@@ -32,3 +32,8 @@ func NewServiceAccountBuilder(namespace, name string) *ServiceAccountBuilder {
 	builder.init(namespace, name, &corev1.ServiceAccount{}, builder)
 	return builder
 }
+
+func (b *ServiceAccountBuilder) SetImagePullSecrets(secrets []corev1.LocalObjectReference) *ServiceAccountBuilder {
+	b.get().ImagePullSecrets = secrets
+	return b
+}
diff --git a/pkg/controller/factory/builder.go b/pkg/controller/factory/builder.go
@@ -505,6 +505,7 @@ func BuildServiceAccount(cluster *appsv1alpha1.Cluster, saName string) *corev1.S
 	wellKnownLabels := constant.GetKBWellKnownLabels(cluster.Spec.ClusterDefRef, cluster.Name, "")
 	return builder.NewServiceAccountBuilder(cluster.Namespace, saName).
 		AddLabelsInMap(wellKnownLabels).
+		SetImagePullSecrets(intctrlutil.BuildImagePullSecrets()).
 		GetObject()
 }
 

diff --git a/pkg/controllerutil/pod_utils.go b/pkg/controllerutil/pod_utils.go
@@ -20,6 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package controllerutil
 
 import (
+	"encoding/json"
 	"fmt"
 	"strconv"
 	"strings"
@@ -33,6 +34,7 @@ import (
 
 	appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1"
 	"github.com/apecloud/kubeblocks/pkg/constant"
+	viper "github.com/apecloud/kubeblocks/pkg/viperx"
 )
 
 const (
@@ -582,3 +584,16 @@ func isContainerFailedAndTimedOut(pod *corev1.Pod, podConditionType corev1.PodCo
 	}
 	return time.Now().After(containerReadyCondition.LastTransitionTime.Add(PodContainerFailedTimeout))
 }
+
+func BuildImagePullSecrets() []corev1.LocalObjectReference {
+	secrets := make([]corev1.LocalObjectReference, 0)
+	secretsVal := viper.GetString(constant.KBImagePullSecrets)
+	if secretsVal == "" {
+		return secrets
+	}
+
+	// we already validate the value of KBImagePullSecrets when start server,
+	// so we can ignore the error here
+	_ = json.Unmarshal([]byte(secretsVal), &secrets)
+	return secrets
+}
diff --git a/pkg/controllerutil/pod_utils_test.go b/pkg/controllerutil/pod_utils_test.go
@@ -38,6 +38,7 @@ import (
 	appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1"
 	"github.com/apecloud/kubeblocks/pkg/constant"
 	testk8s "github.com/apecloud/kubeblocks/pkg/testutil/k8s"
+	viper "github.com/apecloud/kubeblocks/pkg/viperx"
 )
 
 type TestResourceUnit struct {
@@ -607,3 +608,37 @@ var _ = Describe("pod utils", func() {
 		})
 	})
 })
+
+func TestBuildImagePullSecretsByEnv(t *testing.T) {
+	tests := []struct {
+		value    string
+		expected []corev1.LocalObjectReference
+	}{
+		{
+			value:    "",
+			expected: nil,
+		},
+		{
+			value: "[{\"name\":\"test\"}]",
+			expected: []corev1.LocalObjectReference{
+				{
+					Name: "test",
+				},
+			},
+		},
+	}
+
+	Context("test BuildImagePullSecrets", func() {
+		It("Should succeed with no error", func() {
+			for _, t := range tests {
+				viper.Set(constant.KBImagePullSecrets, t.value)
+				secrets := BuildImagePullSecrets()
+				if t.value == "" {
+					Expect(len(secrets)).To(Equal(0))
+				} else {
+					Expect(secrets).To(Equal(t.expected))
+				}
+			}
+		})
+	})
+}