Skip to content

Commit

Permalink
PCP-2154 (#867)
Browse files Browse the repository at this point in the history 
* Update podidentitywebhook.go

* Update podidentitywebhook.go

* remove pdb, update with failure policy: fail

* Support updating pre-existing clusters

* update

* add TolerationOpExists

* feat: add nodeAffinity

* Update pkg/cloud/services/iam/podidentitywebhook.go

Co-authored-by: Pavan Sokke Nagaraj <[email protected]>

* Update podidentitywebhook.go

* Update podidentitywebhook.go

* feat: clean tolerations and node-affinity

* chore: fix typo

* chore: remove PIW image upgrade

---------

Co-authored-by: Pavan Sokke Nagaraj <[email protected]>
  • Loading branch information
@wcrum @pavansokkenagaraj
wcrum and pavansokkenagaraj authored Nov 28, 2023
1 parent 5d756cd commit 7916773
Show file tree
Hide file tree
Showing 2 changed files with 189 additions and 87 deletions.
274 changes: 188 additions & 86 deletions pkg/cloud/services/iam/podidentitywebhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,16 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/client"

"sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
podIdentityWebhookName = "pod-identity-webhook"
podIdentityWebhookImage = "amazon/amazon-eks-pod-identity-webhook:v0.4.0"

labelNodeRoleMaster = "node-role.kubernetes.io/master"
labelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane"
)

func reconcileServiceAccount(ctx context.Context, ns string, remoteClient client.Client) error {
Expand Down Expand Up @@ -155,67 +157,153 @@ func reconcileDeployment(ctx context.Context, ns string, secret *corev1.Secret,
return err
}

if check.UID != "" {
return nil
}

replicas := int32(1)
deployment := &v13.Deployment{
ObjectMeta: objectMeta(podIdentityWebhookName, ns),
Spec: v13.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": podIdentityWebhookName,
nodeAffinity := &corev1.NodeAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{
{
Weight: 10,
Preference: corev1.NodeSelectorTerm{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: labelNodeRoleMaster,
Operator: corev1.NodeSelectorOpExists,
}, {
Key: labelNodeRoleControlPlane,
Operator: corev1.NodeSelectorOpExists,
},
},
},
}, {
Weight: 10,
Preference: corev1.NodeSelectorTerm{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: labelNodeRoleControlPlane,
Operator: corev1.NodeSelectorOpExists,
},
},
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
},
}

tolerations := []corev1.Toleration{
{
Key: labelNodeRoleControlPlane,
Effect: corev1.TaintEffectNoSchedule,
}, {
Key: labelNodeRoleMaster,
Effect: corev1.TaintEffectNoSchedule,
},
}

if check.UID == "" {
replicas := int32(1)

deployment := &v13.Deployment{
ObjectMeta: objectMeta(podIdentityWebhookName, ns),
Spec: v13.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": podIdentityWebhookName,
},
},
Spec: corev1.PodSpec{
ServiceAccountName: podIdentityWebhookName,
Containers: []corev1.Container{
{
Name: podIdentityWebhookName,
Image: podIdentityWebhookImage,
ImagePullPolicy: corev1.PullIfNotPresent,
VolumeMounts: []corev1.VolumeMount{
{
Name: "webhook-certs",
MountPath: "/etc/webhook/certs/",
ReadOnly: false,
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": podIdentityWebhookName,
},
},
Spec: corev1.PodSpec{
Affinity: &corev1.Affinity{NodeAffinity: nodeAffinity},
Tolerations: tolerations,
ServiceAccountName: podIdentityWebhookName,
Containers: []corev1.Container{
{
Name: podIdentityWebhookName,
Image: podIdentityWebhookImage,
ImagePullPolicy: corev1.PullIfNotPresent,
VolumeMounts: []corev1.VolumeMount{
{
Name: "webhook-certs",
MountPath: "/etc/webhook/certs/",
ReadOnly: false,
},
},
Command: []string{
"/webhook",
"--in-cluster=false",
"--namespace=" + ns,
"--service-name=" + podIdentityWebhookName,
"--annotation-prefix=eks.amazonaws.com",
"--token-audience=sts.amazonaws.com",
"--logtostderr",
},
},
Command: []string{
"/webhook",
"--in-cluster=false",
"--namespace=" + ns,
"--service-name=" + podIdentityWebhookName,
"--annotation-prefix=eks.amazonaws.com",
"--token-audience=sts.amazonaws.com",
"--logtostderr",
},
},
},
Volumes: []corev1.Volume{
{
Name: "webhook-certs",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secret.Name,
Volumes: []corev1.Volume{
{
Name: "webhook-certs",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secret.Name,
},
},
},
},
},
},
},
},
}

return remoteClient.Create(ctx, deployment)
}

needsUpdate := false
if check.Spec.Template.Spec.Affinity == nil {
check.Spec.Template.Spec.Affinity = &corev1.Affinity{}
}
if check.Spec.Template.Spec.Affinity.NodeAffinity == nil {
check.Spec.Template.Spec.Affinity.NodeAffinity = &corev1.NodeAffinity{}
}

for _, aff := range nodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution {
found := false
for _, a := range check.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution {
if len(a.Preference.MatchExpressions) == len(aff.Preference.MatchExpressions) {
for _, e := range a.Preference.MatchExpressions {
for _, e2 := range aff.Preference.MatchExpressions {
if e.Key == e2.Key && e.Operator == e2.Operator {
found = true
}
}
}
}
}
if !found {
check.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(check.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution, aff)
needsUpdate = true
}
}

for _, tol := range tolerations {
found := false
for _, t := range check.Spec.Template.Spec.Tolerations {
if t.Key == tol.Key && t.Effect == tol.Effect {
found = true
}
}
if !found {
check.Spec.Template.Spec.Tolerations = append(check.Spec.Template.Spec.Tolerations, tol)
needsUpdate = true
}
}

if needsUpdate {
return remoteClient.Update(ctx, check)
}

return remoteClient.Create(ctx, deployment)
return nil
}

func reconcileMutatingWebHook(ctx context.Context, ns string, secret *corev1.Secret, remoteClient client.Client) error {
Expand All @@ -227,50 +315,64 @@ func reconcileMutatingWebHook(ctx context.Context, ns string, secret *corev1.Sec
return err
}

if check.UID != "" {
return nil
}

caBundle, ok := secret.Data["ca.crt"]
if !ok {
return errors.New("no CA certificate for the pod identity webhook certificate")
}

mwhMeta := objectMeta(podIdentityWebhookName, ns)
fail := v14.Ignore
none := v14.SideEffectClassNone
mutate := "/mutate"
mwh := &v14.MutatingWebhookConfiguration{
ObjectMeta: mwhMeta,
Webhooks: []v14.MutatingWebhook{
{
Name: podIdentityWebhookName + ".amazonaws.com",
FailurePolicy: &fail,
ClientConfig: v14.WebhookClientConfig{
Service: &v14.ServiceReference{
Name: podIdentityWebhookName,
Namespace: ns,
Path: &mutate,
whFailurePolicyFail := v14.Fail
mutatingWebhookFQN := podIdentityWebhookName + ".amazonaws.com"

if check.UID == "" {
caBundle, ok := secret.Data["ca.crt"]
if !ok {
return errors.New("no CA certificate for the pod identity webhook certificate")
}

mwhMeta := objectMeta(podIdentityWebhookName, ns)
none := v14.SideEffectClassNone
mutate := "/mutate"
mwh := &v14.MutatingWebhookConfiguration{
ObjectMeta: mwhMeta,
Webhooks: []v14.MutatingWebhook{
{
Name: mutatingWebhookFQN,
FailurePolicy: &whFailurePolicyFail,
ClientConfig: v14.WebhookClientConfig{
Service: &v14.ServiceReference{
Name: podIdentityWebhookName,
Namespace: ns,
Path: &mutate,
},
CABundle: caBundle,
},
CABundle: caBundle,
},
Rules: []v14.RuleWithOperations{
{
Operations: []v14.OperationType{v14.Create},
Rule: v14.Rule{
APIGroups: []string{""},
APIVersions: []string{"v1"},
Resources: []string{"pods"},
Rules: []v14.RuleWithOperations{
{
Operations: []v14.OperationType{v14.Create},
Rule: v14.Rule{
APIGroups: []string{""},
APIVersions: []string{"v1"},
Resources: []string{"pods"},
},
},
},
SideEffects: &none,
AdmissionReviewVersions: []string{"v1beta1"},
},
SideEffects: &none,
AdmissionReviewVersions: []string{"v1beta1"},
},
},
}

return remoteClient.Create(ctx, mwh)
}

needsUpdate := false
for i := range check.Webhooks {
if check.Webhooks[i].Name == mutatingWebhookFQN && (check.Webhooks[i].FailurePolicy == nil || *check.Webhooks[i].FailurePolicy != whFailurePolicyFail) {
check.Webhooks[i].FailurePolicy = &whFailurePolicyFail
needsUpdate = true
}
}

return remoteClient.Create(ctx, mwh)
if needsUpdate {
return remoteClient.Update(ctx, check)
}

return nil
}

// reconcilePodIdentityWebhookComponents will create sa, cr, crb, service, deployment and a mutating webhook in kube-system. The
Expand Down Expand Up @@ -319,8 +421,8 @@ func objectMeta(name, namespace string) metav1.ObjectMeta {
return meta
}

// reconcileCertifcateSecret takes a secret and moves it to the workload cluster.
func reconcileCertifcateSecret(ctx context.Context, cert *corev1.Secret, remoteClient client.Client) error {
// reconcileCertificateSecret takes a secret and moves it to the workload cluster.
func reconcileCertificateSecret(ctx context.Context, cert *corev1.Secret, remoteClient client.Client) error {
// check if the secret was created by cert-manager
certCheck := &corev1.Secret{}
if err := remoteClient.Get(ctx, types.NamespacedName{
Expand Down
2 changes: 1 addition & 1 deletion pkg/cloud/services/iam/reconcilers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func (s *Service) reconcilePodIdentityWebhook(ctx context.Context) error {
}

// switch it to kube-system and move it to the remote cluster
if err := reconcileCertifcateSecret(ctx, certSecret, remoteClient); err != nil {
if err := reconcileCertificateSecret(ctx, certSecret, remoteClient); err != nil {
return err
}

Expand Down

0 comments on commit 7916773

Please sign in to comment.