Skip to content

Commit

Permalink
Add support for ETCD encryption in Cloudstack
Browse files Browse the repository at this point in the history
  • Loading branch information
@abhinavmpandey08
abhinavmpandey08 committed Sep 21, 2023
1 parent b128b35 commit 2325457
Show file tree
Hide file tree
Showing 10 changed files with 910 additions and 0 deletions.
10 changes: 10 additions & 0 deletions pkg/clusterapi/extraargs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,16 @@ func AwsIamAuthExtraArgs(awsiam *v1alpha1.AWSIamConfig) ExtraArgs {
return args
}

// EtcdEncryptionExtraArgs takes a list of EtcdEncryption configs and returns the relevant API server extra args if it's not nil or empty.
func EtcdEncryptionExtraArgs(config *[]v1alpha1.EtcdEncryption) ExtraArgs {
args := ExtraArgs{}
if config == nil || len(*config) == 0 {
return args
}
args.AddIfNotEmpty("encryption-provider-config", "/etc/kubernetes/enc/encryption-config.yaml")
return args
}

// FeatureGatesExtraArgs takes a list of features with the value and returns it in the proper format
// Example FeatureGatesExtraArgs("ServiceLoadBalancerClass=true").
func FeatureGatesExtraArgs(features ...string) ExtraArgs {
Expand Down
103 changes: 103 additions & 0 deletions pkg/clusterapi/extraargs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,109 @@ func TestNodeCIDRMaskExtraArgs(t *testing.T) {
}
}

func TestEtcdEncryptionExtraArgs(t *testing.T) {
tests := []struct {
name string
etcdEncryption *[]v1alpha1.EtcdEncryption
want clusterapi.ExtraArgs
}{
{
name: "nil config",
etcdEncryption: nil,
want: clusterapi.ExtraArgs{},
},
{
name: "empty config",
etcdEncryption: &[]v1alpha1.EtcdEncryption{},
want: clusterapi.ExtraArgs{},
},
{
name: "one config",
etcdEncryption: &[]v1alpha1.EtcdEncryption{
{
Providers: []v1alpha1.EtcdEncryptionProvider{
{
KMS: &v1alpha1.KMS{
Name: "config1",
SocketListenAddress: "unix:///var/run/kmsplugin/socket1-new.sock",
},
},
{
KMS: &v1alpha1.KMS{
Name: "config2",
SocketListenAddress: "unix:///var/run/kmsplugin/socket1-old.sock",
},
},
},
Resources: []string{
"secrets",
"crd1.anywhere.eks.amazonsaws.com",
},
},
},
want: clusterapi.ExtraArgs{
"encryption-provider-config": "/etc/kubernetes/enc/encryption-config.yaml",
},
},
{
name: "multiple configs",
etcdEncryption: &[]v1alpha1.EtcdEncryption{
{
Providers: []v1alpha1.EtcdEncryptionProvider{
{
KMS: &v1alpha1.KMS{
Name: "config1",
SocketListenAddress: "unix:///var/run/kmsplugin/socket1-new.sock",
},
},
{
KMS: &v1alpha1.KMS{
Name: "config2",
SocketListenAddress: "unix:///var/run/kmsplugin/socket1-old.sock",
},
},
},
Resources: []string{
"secrets",
"crd1.anywhere.eks.amazonsaws.com",
},
},
{
Providers: []v1alpha1.EtcdEncryptionProvider{
{
KMS: &v1alpha1.KMS{
Name: "config3",
SocketListenAddress: "unix:///var/run/kmsplugin/socket2-new.sock",
},
},
{
KMS: &v1alpha1.KMS{
Name: "config4",
SocketListenAddress: "unix:///var/run/kmsplugin/socket2-old.sock",
},
},
},
Resources: []string{
"configmaps",
"crd2.anywhere.eks.amazonsaws.com",
},
},
},
want: clusterapi.ExtraArgs{
"encryption-provider-config": "/etc/kubernetes/enc/encryption-config.yaml",
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(*testing.T) {
if got := clusterapi.EtcdEncryptionExtraArgs(tt.etcdEncryption); !reflect.DeepEqual(got, tt.want) {
t.Errorf("EtcdEncryptionExtraArgs() = %v, want %v", got, tt.want)
}
})
}
}

func TestFeatureGatesExtraArgs(t *testing.T) {
tests := []struct {
testName string
Expand Down
69 changes: 69 additions & 0 deletions pkg/providers/cloudstack/cloudstack_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2600,3 +2600,72 @@ func TestValidateNewSpecMachineConfigNotFound(t *testing.T) {
err := provider.ValidateNewSpec(context.TODO(), &types.Cluster{}, newClusterSpec)
assert.ErrorContains(t, err, "not found")
}

func TestProviderGenerateCAPISpecForUpgradeEtcdEncryption(t *testing.T) {
tests := []struct {
testName string
clusterconfigFile string
wantCPFile string
wantMDFile string
}{
{
testName: "etcd-encryption",
clusterconfigFile: "cluster_etcd_encryption.yaml",
wantCPFile: "testdata/expected_results_encryption_config_cp.yaml",
wantMDFile: "testdata/expected_results_minimal_md.yaml",
},
}
for _, tt := range tests {
t.Run(tt.testName, func(t *testing.T) {
mockCtrl := gomock.NewController(t)
setupContext(t)
ctx := context.Background()
kubectl := mocks.NewMockProviderKubectlClient(mockCtrl)
cluster := &types.Cluster{
Name: "test",
}
bootstrapCluster := &types.Cluster{
Name: "bootstrap-test",
}
clusterSpec := givenClusterSpec(t, tt.clusterconfigFile)
cloudstackDatacenter := &v1alpha1.CloudStackDatacenterConfig{
Spec: v1alpha1.CloudStackDatacenterConfigSpec{},
}
cloudstackMachineConfig := &v1alpha1.CloudStackMachineConfig{
Spec: v1alpha1.CloudStackMachineConfigSpec{
Users: []v1alpha1.UserConfiguration{
{
Name: "capv",
SshAuthorizedKeys: []string{"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ=="},
},
},
},
}

kubectl.EXPECT().GetMachineDeployment(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(workerNodeGroup1MachineDeployment(), nil)
kubectl.EXPECT().GetEksaCluster(ctx, cluster, clusterSpec.Cluster.Name).Return(clusterSpec.Cluster, nil)
kubectl.EXPECT().GetEksaCloudStackDatacenterConfig(ctx, cluster.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackDatacenter, nil)
kubectl.EXPECT().GetEksaCloudStackMachineConfig(ctx, clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackMachineConfig, nil)
kubectl.EXPECT().GetEksaCloudStackMachineConfig(ctx, clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations[0].MachineGroupRef.Name, cluster.KubeconfigFile, clusterSpec.Cluster.Namespace).Return(cloudstackMachineConfig, nil)
datacenterConfig := givenDatacenterConfig(t, tt.clusterconfigFile)
validator := givenWildcardValidator(mockCtrl, clusterSpec)
provider := newProviderWithKubectl(t, datacenterConfig, clusterSpec.Cluster, kubectl, validator)
if provider == nil {
t.Fatalf("provider object is nil")
}

err := provider.SetupAndValidateCreateCluster(ctx, clusterSpec)
if err != nil {
t.Fatalf("failed to setup and validate: %v", err)
}

cp, md, err := provider.GenerateCAPISpecForUpgrade(context.Background(), bootstrapCluster, cluster, clusterSpec, clusterSpec.DeepCopy())
if err != nil {
t.Fatalf("failed to generate cluster api spec contents: %v", err)
}

test.AssertContentToFile(t, string(cp), tt.wantCPFile)
test.AssertContentToFile(t, string(md), tt.wantMDFile)
})
}
}
16 changes: 16 additions & 0 deletions pkg/providers/cloudstack/config/template-cp.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,16 @@ spec:
name: awsiamcert
readOnly: false
{{- end}}
{{- if .encryptionProviderConfig }}
- hostPath: /etc/kubernetes/enc
mountPath: /etc/kubernetes/enc
name: encryption-config
readOnly: false
- hostPath: /var/run/kmsplugin/
mountPath: /var/run/kmsplugin/
name: kms-plugin
readOnly: false
{{- end }}
controllerManager:
extraArgs:
cloud-provider: external
Expand All @@ -133,6 +143,12 @@ spec:
{{ .schedulerExtraArgs.ToYaml | indent 10 }}
{{- end }}
files:
{{- if .encryptionProviderConfig }}
- content: |
{{ .encryptionProviderConfig | indent 8}}
owner: root:root
path: /etc/kubernetes/enc/encryption-config.yaml
{{- end }}
{{- if .cloudstackKubeVip}}
- content: |
apiVersion: v1
Expand Down
10 changes: 10 additions & 0 deletions pkg/providers/cloudstack/template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,9 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
apiServerExtraArgs := clusterapi.OIDCToExtraArgs(clusterSpec.OIDCConfig).
Append(clusterapi.AwsIamAuthExtraArgs(clusterSpec.AWSIamConfig)).
Append(clusterapi.PodIAMAuthExtraArgs(clusterSpec.Cluster.Spec.PodIAMConfig)).
Append(clusterapi.EtcdEncryptionExtraArgs(clusterSpec.Cluster.Spec.EtcdEncryption)).
Append(sharedExtraArgs)

controllerManagerExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
Append(clusterapi.NodeCIDRMaskExtraArgs(&clusterSpec.Cluster.Spec.ClusterNetwork))

Expand Down Expand Up @@ -236,6 +238,14 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
values["maxSurge"] = clusterSpec.Cluster.Spec.ControlPlaneConfiguration.UpgradeRolloutStrategy.RollingUpdate.MaxSurge
}

if clusterSpec.Cluster.Spec.EtcdEncryption != nil && len(*clusterSpec.Cluster.Spec.EtcdEncryption) != 0 {
conf, err := common.GenerateKMSEncryptionConfiguration(clusterSpec.Cluster.Spec.EtcdEncryption)
if err != nil {
return nil, err
}

Check warning on line 245 in pkg/providers/cloudstack/template.go

View check run for this annotation

Codecov / codecov/patch

pkg/providers/cloudstack/template.go#L244-L245 

Added lines #L244 - L245 were not covered by tests
values["encryptionProviderConfig"] = conf
}

return values, nil
}

Expand Down
81 changes: 81 additions & 0 deletions pkg/providers/cloudstack/testdata/cluster_etcd_encryption.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
apiVersion: anywhere.eks.amazonaws.com/v1alpha1
kind: Cluster
metadata:
name: test
namespace: test-namespace
spec:
clusterNetwork:
cni: cilium
pods:
cidrBlocks:
- 192.168.0.0/16
services:
cidrBlocks:
- 10.96.0.0/12
controlPlaneConfiguration:
count: 3
endpoint:
host: 1.2.3.4
machineGroupRef:
kind: CloudStackMachineConfig
name: test
datacenterRef:
kind: CloudStackDatacenterConfig
name: test
kubernetesVersion: "1.21"
etcdEncryption:
- providers:
- kms:
name: config1
socketListenAddress: unix:///var/run/kmsplugin/socket1-new.sock
- kms:
name: config2
socketListenAddress: unix:///var/run/kmsplugin/socket1-old.sock
resources:
- secrets
- resource1.anywhere.eks.amazonsaws.com
- providers:
- kms:
name: config3
socketListenAddress: unix:///var/run/kmsplugin/socket2-new.sock
- kms:
name: config4
socketListenAddress: unix:///var/run/kmsplugin/socket2-old.sock
resources:
- configmaps
- resource2.anywhere.eks.amazonsaws.com
workerNodeGroupConfigurations:
- count: 3
machineGroupRef:
kind: CloudStackMachineConfig
name: test
---
apiVersion: anywhere.eks.amazonaws.com/v1alpha1
kind: CloudStackDatacenterConfig
metadata:
name: test
namespace: test-namespace
spec:
account: "admin"
domain: "domain1"
zones:
- name: "zone1"
network:
name: "net1"
managementApiEndpoint: "http://127.16.0.1:8080/client/api"
---
apiVersion: anywhere.eks.amazonaws.com/v1alpha1
kind: CloudStackMachineConfig
metadata:
name: test
namespace: test-namespace
spec:
computeOffering:
name: "m4-large"
users:
- name: "mySshUsername"
sshAuthorizedKeys:
- "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== [email protected]"
template:
name: "centos7-k8s-118"
---
Loading

0 comments on commit 2325457

Please sign in to comment.