diff --git a/pkg/clusterapi/extraargs.go b/pkg/clusterapi/extraargs.go index 68dff6dc8c5b6..fb54eb95e1dd3 100644 --- a/pkg/clusterapi/extraargs.go +++ b/pkg/clusterapi/extraargs.go @@ -43,6 +43,16 @@ func AwsIamAuthExtraArgs(awsiam *v1alpha1.AWSIamConfig) ExtraArgs { return args } +// EtcdEncryptionExtraArgs takes a list of EtcdEncryption configs and returns the relevant API server extra args if it's not nil or empty. +func EtcdEncryptionExtraArgs(config *[]v1alpha1.EtcdEncryption) ExtraArgs { + args := ExtraArgs{} + if config == nil || len(*config) == 0 { + return args + } + args.AddIfNotEmpty("encryption-provider-config", "/etc/kubernetes/enc/encryption-config.yaml") + return args +} + // FeatureGatesExtraArgs takes a list of features with the value and returns it in the proper format // Example FeatureGatesExtraArgs("ServiceLoadBalancerClass=true"). func FeatureGatesExtraArgs(features ...string) ExtraArgs { diff --git a/pkg/clusterapi/extraargs_test.go b/pkg/clusterapi/extraargs_test.go index d5402550e1e97..50c8f0404d618 100644 --- a/pkg/clusterapi/extraargs_test.go +++ b/pkg/clusterapi/extraargs_test.go @@ -519,6 +519,109 @@ func TestNodeCIDRMaskExtraArgs(t *testing.T) { } } +func TestEtcdEncryptionExtraArgs(t *testing.T) { + tests := []struct { + name string + etcdEncryption *[]v1alpha1.EtcdEncryption + want clusterapi.ExtraArgs + }{ + { + name: "nil config", + etcdEncryption: nil, + want: clusterapi.ExtraArgs{}, + }, + { + name: "empty config", + etcdEncryption: &[]v1alpha1.EtcdEncryption{}, + want: clusterapi.ExtraArgs{}, + }, + { + name: "one config", + etcdEncryption: &[]v1alpha1.EtcdEncryption{ + { + Providers: []v1alpha1.EtcdEncryptionProvider{ + { + KMS: &v1alpha1.KMS{ + Name: "config1", + SocketListenAddress: "unix:///var/run/kmsplugin/socket1-new.sock", + }, + }, + { + KMS: &v1alpha1.KMS{ + Name: "config2", + SocketListenAddress: "unix:///var/run/kmsplugin/socket1-old.sock", + }, + }, + }, + Resources: []string{ + "secrets", + "crd1.anywhere.eks.amazonsaws.com", + }, + }, + }, + want: clusterapi.ExtraArgs{ + "encryption-provider-config": "/etc/kubernetes/enc/encryption-config.yaml", + }, + }, + { + name: "multiple configs", + etcdEncryption: &[]v1alpha1.EtcdEncryption{ + { + Providers: []v1alpha1.EtcdEncryptionProvider{ + { + KMS: &v1alpha1.KMS{ + Name: "config1", + SocketListenAddress: "unix:///var/run/kmsplugin/socket1-new.sock", + }, + }, + { + KMS: &v1alpha1.KMS{ + Name: "config2", + SocketListenAddress: "unix:///var/run/kmsplugin/socket1-old.sock", + }, + }, + }, + Resources: []string{ + "secrets", + "crd1.anywhere.eks.amazonsaws.com", + }, + }, + { + Providers: []v1alpha1.EtcdEncryptionProvider{ + { + KMS: &v1alpha1.KMS{ + Name: "config3", + SocketListenAddress: "unix:///var/run/kmsplugin/socket2-new.sock", + }, + }, + { + KMS: &v1alpha1.KMS{ + Name: "config4", + SocketListenAddress: "unix:///var/run/kmsplugin/socket2-old.sock", + }, + }, + }, + Resources: []string{ + "configmaps", + "crd2.anywhere.eks.amazonsaws.com", + }, + }, + }, + want: clusterapi.ExtraArgs{ + "encryption-provider-config": "/etc/kubernetes/enc/encryption-config.yaml", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + if got := clusterapi.EtcdEncryptionExtraArgs(tt.etcdEncryption); !reflect.DeepEqual(got, tt.want) { + t.Errorf("EtcdEncryptionExtraArgs() = %v, want %v", got, tt.want) + } + }) + } +} + func TestFeatureGatesExtraArgs(t *testing.T) { tests := []struct { testName string diff --git a/pkg/providers/cloudstack/cloudstack_test.go b/pkg/providers/cloudstack/cloudstack_test.go index 39027664d8a25..695a9ce2d5e37 100644 --- a/pkg/providers/cloudstack/cloudstack_test.go +++ b/pkg/providers/cloudstack/cloudstack_test.go @@ -2600,3 +2600,72 @@ func TestValidateNewSpecMachineConfigNotFound(t *testing.T) { err := provider.ValidateNewSpec(context.TODO(), &types.Cluster{}, newClusterSpec) assert.ErrorContains(t, err, "not found") } + +func TestProviderGenerateCAPISpecForUpgradeEtcdEncryption(t *testing.T) { + tests := []struct { + testName string + clusterconfigFile string + wantCPFile string + wantMDFile string + }{ + { + testName: "etcd-encryption", + clusterconfigFile: "cluster_etcd_encryption.yaml", + wantCPFile: "testdata/expected_results_encryption_config_cp.yaml", + wantMDFile: "testdata/expected_results_minimal_md.yaml", + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + mockCtrl := gomock.NewController(t) + setupContext(t) + ctx := context.Background() + kubectl := mocks.NewMockProviderKubectlClient(mockCtrl) + cluster := &types.Cluster{ + Name: "test", + } + bootstrapCluster := &types.Cluster{ + Name: "bootstrap-test", + } + clusterSpec := givenClusterSpec(t, tt.clusterconfigFile) + cloudstackDatacenter := &v1alpha1.CloudStackDatacenterConfig{ + Spec: v1alpha1.CloudStackDatacenterConfigSpec{}, + } + cloudstackMachineConfig := &v1alpha1.CloudStackMachineConfig{ + Spec: v1alpha1.CloudStackMachineConfigSpec{ + Users: []v1alpha1.UserConfiguration{ + { + Name: "capv", + SshAuthorizedKeys: []string{"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ=="}, + }, + }, + }, + } + + kubectl.EXPECT().GetMachineDeployment(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(workerNodeGroup1MachineDeployment(), nil) + kubectl.EXPECT().GetEksaCluster(ctx, cluster, clusterSpec.Cluster.Name).Return(clusterSpec.Cluster, nil) + kubectl.EXPECT().GetEksaCloudStackDatacenterConfig(ctx, cluster.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackDatacenter, nil) + kubectl.EXPECT().GetEksaCloudStackMachineConfig(ctx, clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackMachineConfig, nil) + kubectl.EXPECT().GetEksaCloudStackMachineConfig(ctx, clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations[0].MachineGroupRef.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackMachineConfig, nil) + datacenterConfig := givenDatacenterConfig(t, tt.clusterconfigFile) + validator := givenWildcardValidator(mockCtrl, clusterSpec) + provider := newProviderWithKubectl(t, datacenterConfig, clusterSpec.Cluster, kubectl, validator) + if provider == nil { + t.Fatalf("provider object is nil") + } + + err := provider.SetupAndValidateCreateCluster(ctx, clusterSpec) + if err != nil { + t.Fatalf("failed to setup and validate: %v", err) + } + + cp, md, err := provider.GenerateCAPISpecForUpgrade(context.Background(), bootstrapCluster, cluster, clusterSpec, clusterSpec.DeepCopy()) + if err != nil { + t.Fatalf("failed to generate cluster api spec contents: %v", err) + } + + test.AssertContentToFile(t, string(cp), tt.wantCPFile) + test.AssertContentToFile(t, string(md), tt.wantMDFile) + }) + } +} diff --git a/pkg/providers/cloudstack/config/template-cp.yaml b/pkg/providers/cloudstack/config/template-cp.yaml index 8d96635d46723..e96266b475c31 100644 --- a/pkg/providers/cloudstack/config/template-cp.yaml +++ b/pkg/providers/cloudstack/config/template-cp.yaml @@ -119,6 +119,16 @@ spec: name: awsiamcert readOnly: false {{- end}} +{{- if .encryptionProviderConfig }} + - hostPath: /etc/kubernetes/enc + mountPath: /etc/kubernetes/enc + name: encryption-config + readOnly: false + - hostPath: /var/run/kmsplugin/ + mountPath: /var/run/kmsplugin/ + name: kms-plugin + readOnly: false +{{- end }} controllerManager: extraArgs: cloud-provider: external @@ -133,6 +143,12 @@ spec: {{ .schedulerExtraArgs.ToYaml | indent 10 }} {{- end }} files: +{{- if .encryptionProviderConfig }} + - content: | +{{ .encryptionProviderConfig | indent 8}} + owner: root:root + path: /etc/kubernetes/enc/encryption-config.yaml +{{- end }} {{- if .cloudstackKubeVip}} - content: | apiVersion: v1 diff --git a/pkg/providers/cloudstack/template.go b/pkg/providers/cloudstack/template.go index d152b7c828796..78945e855ce82 100644 --- a/pkg/providers/cloudstack/template.go +++ b/pkg/providers/cloudstack/template.go @@ -116,7 +116,9 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro apiServerExtraArgs := clusterapi.OIDCToExtraArgs(clusterSpec.OIDCConfig). Append(clusterapi.AwsIamAuthExtraArgs(clusterSpec.AWSIamConfig)). Append(clusterapi.PodIAMAuthExtraArgs(clusterSpec.Cluster.Spec.PodIAMConfig)). + Append(clusterapi.EtcdEncryptionExtraArgs(clusterSpec.Cluster.Spec.EtcdEncryption)). Append(sharedExtraArgs) + controllerManagerExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs(). Append(clusterapi.NodeCIDRMaskExtraArgs(&clusterSpec.Cluster.Spec.ClusterNetwork)) @@ -236,6 +238,14 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro values["maxSurge"] = clusterSpec.Cluster.Spec.ControlPlaneConfiguration.UpgradeRolloutStrategy.RollingUpdate.MaxSurge } + if clusterSpec.Cluster.Spec.EtcdEncryption != nil && len(*clusterSpec.Cluster.Spec.EtcdEncryption) != 0 { + conf, err := common.GenerateKMSEncryptionConfiguration(clusterSpec.Cluster.Spec.EtcdEncryption) + if err != nil { + return nil, err + } + values["encryptionProviderConfig"] = conf + } + return values, nil } diff --git a/pkg/providers/cloudstack/testdata/cluster_etcd_encryption.yaml b/pkg/providers/cloudstack/testdata/cluster_etcd_encryption.yaml new file mode 100644 index 0000000000000..b12062c184dfa --- /dev/null +++ b/pkg/providers/cloudstack/testdata/cluster_etcd_encryption.yaml @@ -0,0 +1,81 @@ +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: Cluster +metadata: + name: test + namespace: test-namespace +spec: + clusterNetwork: + cni: cilium + pods: + cidrBlocks: + - 192.168.0.0/16 + services: + cidrBlocks: + - 10.96.0.0/12 + controlPlaneConfiguration: + count: 3 + endpoint: + host: 1.2.3.4 + machineGroupRef: + kind: CloudStackMachineConfig + name: test + datacenterRef: + kind: CloudStackDatacenterConfig + name: test + kubernetesVersion: "1.21" + etcdEncryption: + - providers: + - kms: + name: config1 + socketListenAddress: unix:///var/run/kmsplugin/socket1-new.sock + - kms: + name: config2 + socketListenAddress: unix:///var/run/kmsplugin/socket1-old.sock + resources: + - secrets + - resource1.anywhere.eks.amazonsaws.com + - providers: + - kms: + name: config3 + socketListenAddress: unix:///var/run/kmsplugin/socket2-new.sock + - kms: + name: config4 + socketListenAddress: unix:///var/run/kmsplugin/socket2-old.sock + resources: + - configmaps + - resource2.anywhere.eks.amazonsaws.com + workerNodeGroupConfigurations: + - count: 3 + machineGroupRef: + kind: CloudStackMachineConfig + name: test +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: CloudStackDatacenterConfig +metadata: + name: test + namespace: test-namespace +spec: + account: "admin" + domain: "domain1" + zones: + - name: "zone1" + network: + name: "net1" + managementApiEndpoint: "http://127.16.0.1:8080/client/api" +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: CloudStackMachineConfig +metadata: + name: test + namespace: test-namespace +spec: + computeOffering: + name: "m4-large" + users: + - name: "mySshUsername" + sshAuthorizedKeys: + - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== testemail@test.com" + template: + name: "centos7-k8s-118" +--- diff --git a/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp.yaml b/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp.yaml new file mode 100644 index 0000000000000..51752b7b8e3eb --- /dev/null +++ b/pkg/providers/cloudstack/testdata/expected_results_encryption_config_cp.yaml @@ -0,0 +1,420 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: test + name: test + namespace: eksa-system +spec: + clusterNetwork: + pods: + cidrBlocks: [192.168.0.0/16] + services: + cidrBlocks: [10.96.0.0/12] + controlPlaneEndpoint: + host: 1.2.3.4 + port: 6443 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: test + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta3 + kind: CloudStackCluster + name: test +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta3 +kind: CloudStackCluster +metadata: + name: test + namespace: eksa-system +spec: + controlPlaneEndpoint: + host: 1.2.3.4 + port: 6443 + failureDomains: + - name: default-az-0 + zone: + id: + name: zone1 + network: + id: + name: net1 + domain: domain1 + account: admin + acsEndpoint: + name: global + namespace: eksa-system +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: test + namespace: eksa-system +spec: + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta3 + kind: CloudStackMachineTemplate + name: test-control-plane-template-1234567890000 + kubeadmConfigSpec: + clusterConfiguration: + imageRepository: public.ecr.aws/eks-distro/kubernetes + etcd: + local: + imageRepository: public.ecr.aws/eks-distro/etcd-io + imageTag: v3.4.16-eks-1-21-4 + extraArgs: + cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + dns: + imageRepository: public.ecr.aws/eks-distro/coredns + imageTag: v1.8.3-eks-1-21-4 + apiServer: + extraArgs: + cloud-provider: external + audit-policy-file: /etc/kubernetes/audit-policy.yaml + audit-log-path: /var/log/kubernetes/api-audit.log + audit-log-maxage: "30" + audit-log-maxbackup: "10" + audit-log-maxsize: "512" + profiling: "false" + encryption-provider-config: /etc/kubernetes/enc/encryption-config.yaml + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + extraVolumes: + - hostPath: /etc/kubernetes/audit-policy.yaml + mountPath: /etc/kubernetes/audit-policy.yaml + name: audit-policy + pathType: File + readOnly: true + - hostPath: /var/log/kubernetes + mountPath: /var/log/kubernetes + name: audit-log-dir + pathType: DirectoryOrCreate + readOnly: false + - hostPath: /etc/kubernetes/enc + mountPath: /etc/kubernetes/enc + name: encryption-config + readOnly: false + - hostPath: /var/run/kmsplugin/ + mountPath: /var/run/kmsplugin/ + name: kms-plugin + readOnly: false + controllerManager: + extraArgs: + cloud-provider: external + profiling: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + scheduler: + extraArgs: + profiling: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + files: + - content: | + apiVersion: apiserver.config.k8s.io/v1 + kind: EncryptionConfiguration + resources: + - providers: + - kms: + apiVersion: v1 + cachesize: 1000 + endpoint: unix:///var/run/kmsplugin/socket1-new.sock + name: config1 + timeout: 3s + - kms: + apiVersion: v1 + cachesize: 1000 + endpoint: unix:///var/run/kmsplugin/socket1-old.sock + name: config2 + timeout: 3s + - identity: {} + resources: + - secrets + - resource1.anywhere.eks.amazonsaws.com + - providers: + - kms: + apiVersion: v1 + cachesize: 1000 + endpoint: unix:///var/run/kmsplugin/socket2-new.sock + name: config3 + timeout: 3s + - kms: + apiVersion: v1 + cachesize: 1000 + endpoint: unix:///var/run/kmsplugin/socket2-old.sock + name: config4 + timeout: 3s + - identity: {} + resources: + - configmaps + - resource2.anywhere.eks.amazonsaws.com + owner: root:root + path: /etc/kubernetes/enc/encryption-config.yaml + - content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: address + value: 1.2.3.4 + image: public.ecr.aws/l0g8r8j6/kube-vip/kube-vip:v0.3.7-eks-a-v0.0.0-dev-build.158 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + - content: | + apiVersion: audit.k8s.io/v1beta1 + kind: Policy + rules: + # Log aws-auth configmap changes + - level: RequestResponse + namespaces: ["kube-system"] + verbs: ["update", "patch", "delete"] + resources: + - group: "" # core + resources: ["configmaps"] + resourceNames: ["aws-auth"] + omitStages: + - "RequestReceived" + # The following requests were manually identified as high-volume and low-risk, + # so drop them. + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core + resources: ["endpoints", "services", "services/status"] + - level: None + users: ["kubelet"] # legacy kubelet identity + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + userGroups: ["system:nodes"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:serviceaccount:kube-system:endpoint-controller + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["endpoints"] + - level: None + users: ["system:apiserver"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + # Don't log HPA fetching metrics. + - level: None + users: + - system:kube-controller-manager + verbs: ["get", "list"] + resources: + - group: "metrics.k8s.io" + # Don't log these read-only URLs. + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + # Don't log events requests. + - level: None + resources: + - group: "" # core + resources: ["events"] + # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes + - level: Request + users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + - level: Request + userGroups: ["system:nodes"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + # deletecollection calls can be large, don't log responses for expected namespace deletions + - level: Request + users: ["system:serviceaccount:kube-system:namespace-controller"] + verbs: ["deletecollection"] + omitStages: + - "RequestReceived" + # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, + # so only log at the Metadata level. + - level: Metadata + resources: + - group: "" # core + resources: ["secrets", "configmaps"] + - group: authentication.k8s.io + resources: ["tokenreviews"] + omitStages: + - "RequestReceived" + - level: Request + resources: + - group: "" + resources: ["serviceaccounts/token"] + # Get repsonses can be large; skip them. + - level: Request + verbs: ["get", "list", "watch"] + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for known APIs + - level: RequestResponse + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for all other requests. + - level: Metadata + omitStages: + - "RequestReceived" + owner: root:root + path: /etc/kubernetes/audit-policy.yaml + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}' + read-only-port: "0" + anonymous-auth: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + name: "{{ ds.meta_data.hostname }}" + joinConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}' + read-only-port: "0" + anonymous-auth: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + name: "{{ ds.meta_data.hostname }}" + preKubeadmCommands: + - swapoff -a + - hostname "{{ ds.meta_data.hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 {{ ds.meta_data.hostname }}" >>/etc/hosts + - echo "{{ ds.meta_data.hostname }}" >/etc/hostname + useExperimentalRetryJoin: true + users: + - name: mySshUsername + sshAuthorizedKeys: + - 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ==' + sudo: ALL=(ALL) NOPASSWD:ALL + format: cloud-config + replicas: 3 + version: v1.21.2-eks-1-21-4 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta3 +kind: CloudStackMachineTemplate +metadata: + creationTimestamp: null + name: test-control-plane-template-1234567890000 + namespace: eksa-system +spec: + template: + spec: + diskOffering: + customSizeInGB: 0 + device: "" + filesystem: "" + label: "" + mountPath: "" + offering: + name: m4-large + sshKey: "" + template: + name: centos7-k8s-118 + +--- diff --git a/pkg/providers/common/encryption_config_generator.go b/pkg/providers/common/encryption_config_generator.go new file mode 100644 index 0000000000000..3e1536658bcb9 --- /dev/null +++ b/pkg/providers/common/encryption_config_generator.go @@ -0,0 +1,70 @@ +package common + +import ( + "fmt" + "strings" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + config "k8s.io/apiserver/pkg/apis/config/v1" + "sigs.k8s.io/yaml" + + "github.com/aws/eks-anywhere/pkg/api/v1alpha1" + "github.com/aws/eks-anywhere/pkg/utils/ptr" +) + +const ( + encryptionConfigurationKind = "EncryptionConfiguration" + encryptionProviderVersion = "v1" + encryptionProviderNamePrefix = "aws-encryption-provider" +) + +var identityProvider = config.ProviderConfiguration{ + Identity: &config.IdentityConfiguration{}, +} + +// GenerateKMSEncryptionConfiguration takes a list of the EtcdEncryption configs and generates the corresponding Kubernetes EncryptionConfig. +func GenerateKMSEncryptionConfiguration(confs *[]v1alpha1.EtcdEncryption) (string, error) { + if confs == nil || len(*confs) == 0 { + return "", nil + } + + encryptionConf := &config.EncryptionConfiguration{ + TypeMeta: v1.TypeMeta{ + APIVersion: config.SchemeGroupVersion.Identifier(), + Kind: encryptionConfigurationKind, + }, + } + + resourceConfigs := make([]config.ResourceConfiguration, 0, len(*confs)) + for _, conf := range *confs { + providers := []config.ProviderConfiguration{} + for _, provider := range conf.Providers { + provider := config.ProviderConfiguration{ + KMS: &config.KMSConfiguration{ + APIVersion: encryptionProviderVersion, + Name: provider.KMS.Name, + Endpoint: provider.KMS.SocketListenAddress, + CacheSize: ptr.Int32(1000), + Timeout: &v1.Duration{ + Duration: time.Second * 3, + }, + }, + } + providers = append(providers, provider) + } + providers = append(providers, identityProvider) + resourceConfig := config.ResourceConfiguration{ + Resources: conf.Resources, + Providers: providers, + } + resourceConfigs = append(resourceConfigs, resourceConfig) + } + encryptionConf.Resources = resourceConfigs + + marshaledConf, err := yaml.Marshal(encryptionConf) + if err != nil { + return "", fmt.Errorf("marshaling encryption config: %v", err) + } + return strings.Trim(string(marshaledConf), "\n"), nil +} diff --git a/pkg/providers/common/encryption_config_generator_test.go b/pkg/providers/common/encryption_config_generator_test.go new file mode 100644 index 0000000000000..5b8ecc35dcd2d --- /dev/null +++ b/pkg/providers/common/encryption_config_generator_test.go @@ -0,0 +1,94 @@ +package common_test + +import ( + "os" + "testing" + + . "github.com/onsi/gomega" + + "github.com/aws/eks-anywhere/internal/test" + "github.com/aws/eks-anywhere/pkg/api/v1alpha1" + . "github.com/aws/eks-anywhere/pkg/providers/common" +) + +const expectedEncryptionConfig = "testdata/expected_encryption_config.yaml" + +func TestGenerateKMSEncryptionConfigurationEmpty(t *testing.T) { + g := NewWithT(t) + tests := []struct { + name string + config *[]v1alpha1.EtcdEncryption + want string + }{ + { + name: "nil config", + config: nil, + want: "", + }, + { + name: "empty config", + config: &[]v1alpha1.EtcdEncryption{}, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + got, err := GenerateKMSEncryptionConfiguration(tt.config) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func TestGenerateEncryptionConfiguration(t *testing.T) { + encryptionConf := &[]v1alpha1.EtcdEncryption{ + { + Providers: []v1alpha1.EtcdEncryptionProvider{ + { + KMS: &v1alpha1.KMS{ + Name: "config1", + SocketListenAddress: "unix:///var/run/kmsplugin/socket1-new.sock", + }, + }, + { + KMS: &v1alpha1.KMS{ + Name: "config2", + SocketListenAddress: "unix:///var/run/kmsplugin/socket1-old.sock", + }, + }, + }, + Resources: []string{ + "secrets", + "crd1.anywhere.eks.amazonsaws.com", + }, + }, + { + Providers: []v1alpha1.EtcdEncryptionProvider{ + { + KMS: &v1alpha1.KMS{ + Name: "config3", + SocketListenAddress: "unix:///var/run/kmsplugin/socket2-new.sock", + }, + }, + { + KMS: &v1alpha1.KMS{ + Name: "config4", + SocketListenAddress: "unix:///var/run/kmsplugin/socket2-old.sock", + }, + }, + }, + Resources: []string{ + "configmaps", + "crd2.anywhere.eks.amazonsaws.com", + }, + }, + } + + conf, err := GenerateKMSEncryptionConfiguration(encryptionConf) + if err != nil { + t.Fatal(err) + } + _ = os.WriteFile("test.out", []byte(conf), os.ModePerm) + test.AssertContentToFile(t, conf, expectedEncryptionConfig) +} diff --git a/pkg/providers/common/testdata/expected_encryption_config.yaml b/pkg/providers/common/testdata/expected_encryption_config.yaml new file mode 100755 index 0000000000000..31d301e1adb59 --- /dev/null +++ b/pkg/providers/common/testdata/expected_encryption_config.yaml @@ -0,0 +1,37 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: +- providers: + - kms: + apiVersion: v1 + cachesize: 1000 + endpoint: unix:///var/run/kmsplugin/socket1-new.sock + name: config1 + timeout: 3s + - kms: + apiVersion: v1 + cachesize: 1000 + endpoint: unix:///var/run/kmsplugin/socket1-old.sock + name: config2 + timeout: 3s + - identity: {} + resources: + - secrets + - crd1.anywhere.eks.amazonsaws.com +- providers: + - kms: + apiVersion: v1 + cachesize: 1000 + endpoint: unix:///var/run/kmsplugin/socket2-new.sock + name: config3 + timeout: 3s + - kms: + apiVersion: v1 + cachesize: 1000 + endpoint: unix:///var/run/kmsplugin/socket2-old.sock + name: config4 + timeout: 3s + - identity: {} + resources: + - configmaps + - crd2.anywhere.eks.amazonsaws.com \ No newline at end of file