diff --git a/pkg/cloud/services/iam/podidentitywebhook.go b/pkg/cloud/services/iam/podidentitywebhook.go index 7ba6f86247..4a67a9c9a0 100644 --- a/pkg/cloud/services/iam/podidentitywebhook.go +++ b/pkg/cloud/services/iam/podidentitywebhook.go @@ -12,14 +12,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( podIdentityWebhookName = "pod-identity-webhook" podIdentityWebhookImage = "amazon/amazon-eks-pod-identity-webhook:v0.4.0" + + labelNodeRoleMaster = "node-role.kubernetes.io/master" + labelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane" ) func reconcileServiceAccount(ctx context.Context, ns string, remoteClient client.Client) error { @@ -155,67 +157,153 @@ func reconcileDeployment(ctx context.Context, ns string, secret *corev1.Secret, return err } - if check.UID != "" { - return nil - } - - replicas := int32(1) - deployment := &v13.Deployment{ - ObjectMeta: objectMeta(podIdentityWebhookName, ns), - Spec: v13.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": podIdentityWebhookName, + nodeAffinity := &corev1.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ + { + Weight: 10, + Preference: corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: labelNodeRoleMaster, + Operator: corev1.NodeSelectorOpExists, + }, { + Key: labelNodeRoleControlPlane, + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + }, { + Weight: 10, + Preference: corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: labelNodeRoleControlPlane, + Operator: corev1.NodeSelectorOpExists, + }, + }, }, }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ + }, + } + + tolerations := []corev1.Toleration{ + { + Key: labelNodeRoleControlPlane, + Effect: corev1.TaintEffectNoSchedule, + }, { + Key: labelNodeRoleMaster, + Effect: corev1.TaintEffectNoSchedule, + }, + } + + if check.UID == "" { + replicas := int32(1) + + deployment := &v13.Deployment{ + ObjectMeta: objectMeta(podIdentityWebhookName, ns), + Spec: v13.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ "app": podIdentityWebhookName, }, }, - Spec: corev1.PodSpec{ - ServiceAccountName: podIdentityWebhookName, - Containers: []corev1.Container{ - { - Name: podIdentityWebhookName, - Image: podIdentityWebhookImage, - ImagePullPolicy: corev1.PullIfNotPresent, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "webhook-certs", - MountPath: "/etc/webhook/certs/", - ReadOnly: false, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": podIdentityWebhookName, + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{NodeAffinity: nodeAffinity}, + Tolerations: tolerations, + ServiceAccountName: podIdentityWebhookName, + Containers: []corev1.Container{ + { + Name: podIdentityWebhookName, + Image: podIdentityWebhookImage, + ImagePullPolicy: corev1.PullIfNotPresent, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "webhook-certs", + MountPath: "/etc/webhook/certs/", + ReadOnly: false, + }, + }, + Command: []string{ + "/webhook", + "--in-cluster=false", + "--namespace=" + ns, + "--service-name=" + podIdentityWebhookName, + "--annotation-prefix=eks.amazonaws.com", + "--token-audience=sts.amazonaws.com", + "--logtostderr", }, - }, - Command: []string{ - "/webhook", - "--in-cluster=false", - "--namespace=" + ns, - "--service-name=" + podIdentityWebhookName, - "--annotation-prefix=eks.amazonaws.com", - "--token-audience=sts.amazonaws.com", - "--logtostderr", }, }, - }, - Volumes: []corev1.Volume{ - { - Name: "webhook-certs", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: secret.Name, + Volumes: []corev1.Volume{ + { + Name: "webhook-certs", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secret.Name, + }, }, }, }, }, }, }, - }, + } + + return remoteClient.Create(ctx, deployment) + } + + needsUpdate := false + if check.Spec.Template.Spec.Affinity == nil { + check.Spec.Template.Spec.Affinity = &corev1.Affinity{} + } + if check.Spec.Template.Spec.Affinity.NodeAffinity == nil { + check.Spec.Template.Spec.Affinity.NodeAffinity = &corev1.NodeAffinity{} + } + + for _, aff := range nodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution { + found := false + for _, a := range check.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution { + if len(a.Preference.MatchExpressions) == len(aff.Preference.MatchExpressions) { + for _, e := range a.Preference.MatchExpressions { + for _, e2 := range aff.Preference.MatchExpressions { + if e.Key == e2.Key && e.Operator == e2.Operator { + found = true + } + } + } + } + } + if !found { + check.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(check.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution, aff) + needsUpdate = true + } + } + + for _, tol := range tolerations { + found := false + for _, t := range check.Spec.Template.Spec.Tolerations { + if t.Key == tol.Key && t.Effect == tol.Effect { + found = true + } + } + if !found { + check.Spec.Template.Spec.Tolerations = append(check.Spec.Template.Spec.Tolerations, tol) + needsUpdate = true + } + } + + if needsUpdate { + return remoteClient.Update(ctx, check) } - return remoteClient.Create(ctx, deployment) + return nil } func reconcileMutatingWebHook(ctx context.Context, ns string, secret *corev1.Secret, remoteClient client.Client) error { @@ -227,50 +315,64 @@ func reconcileMutatingWebHook(ctx context.Context, ns string, secret *corev1.Sec return err } - if check.UID != "" { - return nil - } - - caBundle, ok := secret.Data["ca.crt"] - if !ok { - return errors.New("no CA certificate for the pod identity webhook certificate") - } - - mwhMeta := objectMeta(podIdentityWebhookName, ns) - fail := v14.Ignore - none := v14.SideEffectClassNone - mutate := "/mutate" - mwh := &v14.MutatingWebhookConfiguration{ - ObjectMeta: mwhMeta, - Webhooks: []v14.MutatingWebhook{ - { - Name: podIdentityWebhookName + ".amazonaws.com", - FailurePolicy: &fail, - ClientConfig: v14.WebhookClientConfig{ - Service: &v14.ServiceReference{ - Name: podIdentityWebhookName, - Namespace: ns, - Path: &mutate, + whFailurePolicyFail := v14.Fail + mutatingWebhookFQN := podIdentityWebhookName + ".amazonaws.com" + + if check.UID == "" { + caBundle, ok := secret.Data["ca.crt"] + if !ok { + return errors.New("no CA certificate for the pod identity webhook certificate") + } + + mwhMeta := objectMeta(podIdentityWebhookName, ns) + none := v14.SideEffectClassNone + mutate := "/mutate" + mwh := &v14.MutatingWebhookConfiguration{ + ObjectMeta: mwhMeta, + Webhooks: []v14.MutatingWebhook{ + { + Name: mutatingWebhookFQN, + FailurePolicy: &whFailurePolicyFail, + ClientConfig: v14.WebhookClientConfig{ + Service: &v14.ServiceReference{ + Name: podIdentityWebhookName, + Namespace: ns, + Path: &mutate, + }, + CABundle: caBundle, }, - CABundle: caBundle, - }, - Rules: []v14.RuleWithOperations{ - { - Operations: []v14.OperationType{v14.Create}, - Rule: v14.Rule{ - APIGroups: []string{""}, - APIVersions: []string{"v1"}, - Resources: []string{"pods"}, + Rules: []v14.RuleWithOperations{ + { + Operations: []v14.OperationType{v14.Create}, + Rule: v14.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, }, }, + SideEffects: &none, + AdmissionReviewVersions: []string{"v1beta1"}, }, - SideEffects: &none, - AdmissionReviewVersions: []string{"v1beta1"}, }, - }, + } + + return remoteClient.Create(ctx, mwh) + } + + needsUpdate := false + for i := range check.Webhooks { + if check.Webhooks[i].Name == mutatingWebhookFQN && (check.Webhooks[i].FailurePolicy == nil || *check.Webhooks[i].FailurePolicy != whFailurePolicyFail) { + check.Webhooks[i].FailurePolicy = &whFailurePolicyFail + needsUpdate = true + } } - return remoteClient.Create(ctx, mwh) + if needsUpdate { + return remoteClient.Update(ctx, check) + } + + return nil } // reconcilePodIdentityWebhookComponents will create sa, cr, crb, service, deployment and a mutating webhook in kube-system. The @@ -319,8 +421,8 @@ func objectMeta(name, namespace string) metav1.ObjectMeta { return meta } -// reconcileCertifcateSecret takes a secret and moves it to the workload cluster. -func reconcileCertifcateSecret(ctx context.Context, cert *corev1.Secret, remoteClient client.Client) error { +// reconcileCertificateSecret takes a secret and moves it to the workload cluster. +func reconcileCertificateSecret(ctx context.Context, cert *corev1.Secret, remoteClient client.Client) error { // check if the secret was created by cert-manager certCheck := &corev1.Secret{} if err := remoteClient.Get(ctx, types.NamespacedName{ diff --git a/pkg/cloud/services/iam/reconcilers.go b/pkg/cloud/services/iam/reconcilers.go index 39c9ecda7b..0b03fe0faa 100644 --- a/pkg/cloud/services/iam/reconcilers.go +++ b/pkg/cloud/services/iam/reconcilers.go @@ -43,7 +43,7 @@ func (s *Service) reconcilePodIdentityWebhook(ctx context.Context) error { } // switch it to kube-system and move it to the remote cluster - if err := reconcileCertifcateSecret(ctx, certSecret, remoteClient); err != nil { + if err := reconcileCertificateSecret(ctx, certSecret, remoteClient); err != nil { return err }