From 27f0b320e274865153054181c9f9594f8995084e Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 12 Nov 2024 11:05:31 +0100 Subject: [PATCH 01/68] Replacing the storage class with pv --- pipeline-runner/internal/hash/hash.go | 62 ----------- pipeline-runner/internal/test/utils.go | 10 +- pipeline-runner/steps/applyconfig/step.go | 5 +- pipeline-runner/steps/deploy/step.go | 5 +- .../steps/deployconfig/step_test.go | 6 +- pipeline-runner/steps/internal/hash.go | 43 -------- pkg/apis/deployment/volumemount.go | 83 +++++++------- pkg/apis/deployment/volumemount_test.go | 2 +- .../apis/utils}/hash/encoding.go | 0 pkg/apis/utils/hash/hash.go | 102 ++++++++++++++++++ .../apis/utils}/hash/hash_test.go | 10 +- .../apis/utils}/hash/sha256.go | 0 pkg/apis/utils/storageclass.go | 89 +++++++-------- 13 files changed, 203 insertions(+), 214 deletions(-) delete mode 100644 pipeline-runner/internal/hash/hash.go delete mode 100644 pipeline-runner/steps/internal/hash.go rename {pipeline-runner/internal => pkg/apis/utils}/hash/encoding.go (100%) create mode 100644 pkg/apis/utils/hash/hash.go rename {pipeline-runner/internal => pkg/apis/utils}/hash/hash_test.go (92%) rename {pipeline-runner/internal => pkg/apis/utils}/hash/sha256.go (100%) diff --git a/pipeline-runner/internal/hash/hash.go b/pipeline-runner/internal/hash/hash.go deleted file mode 100644 index c532aa3a0..000000000 --- a/pipeline-runner/internal/hash/hash.go +++ /dev/null @@ -1,62 +0,0 @@ -package hash - -import ( - "encoding/hex" - "fmt" - "strings" -) - -type Algorithm string - -const ( - hashStringSeparator string = "=" -) - -type sumFunc func(in []byte) []byte - -var sumProviders map[Algorithm]sumFunc = map[Algorithm]sumFunc{} - -func registerProvider(alg Algorithm, provider sumFunc) { - sumProviders[alg] = provider -} - -func ToHashString(alg Algorithm, source any) (string, error) { - sum, found := sumProviders[alg] - if !found { - return "", fmt.Errorf("hash algorithm %s not found", alg) - } - - encode, err := getEncoder(source) - if err != nil { - return "", err - } - - b, err := encode(source) - if err != nil { - return "", err - } - - hashBytes := sum(b) - return joinToHashString(alg, hex.EncodeToString(hashBytes)), nil -} - -func CompareWithHashString(source any, targetHashString string) (bool, error) { - alg := extractAlgorithmFromHashString(targetHashString) - sourceHashString, err := ToHashString(alg, source) - if err != nil { - return false, err - } - return targetHashString == sourceHashString, nil -} - -func joinToHashString(alg Algorithm, hash string) string { - return fmt.Sprintf("%s%s%s", alg, hashStringSeparator, hash) -} - -func extractAlgorithmFromHashString(hashString string) Algorithm { - parts := strings.Split(hashString, hashStringSeparator) - if len(parts) != 2 { - return "" - } - return Algorithm(parts[0]) -} diff --git a/pipeline-runner/internal/test/utils.go b/pipeline-runner/internal/test/utils.go index 085456e62..e84050e42 100644 --- a/pipeline-runner/internal/test/utils.go +++ b/pipeline-runner/internal/test/utils.go @@ -3,12 +3,12 @@ package test import ( "context" - "github.com/equinor/radix-operator/pipeline-runner/internal/hash" "github.com/equinor/radix-operator/pipeline-runner/model" pipelineDefaults "github.com/equinor/radix-operator/pipeline-runner/model/defaults" "github.com/equinor/radix-operator/pkg/apis/defaults" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" + hash2 "github.com/equinor/radix-operator/pkg/apis/utils/hash" "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -64,18 +64,18 @@ func CreateBuildSecret(kubeClient kubernetes.Interface, appName string, data map func GetRadixApplicationHash(ra *radixv1.RadixApplication) string { if ra == nil { - hash, _ := hash.ToHashString(hash.SHA256, "0nXSg9l6EUepshGFmolpgV3elB0m8Mv7") + hash, _ := hash2.ToHashString(hash2.SHA256, "0nXSg9l6EUepshGFmolpgV3elB0m8Mv7") return hash } - hash, _ := hash.ToHashString(hash.SHA256, ra.Spec) + hash, _ := hash2.ToHashString(hash2.SHA256, ra.Spec) return hash } func GetBuildSecretHash(secret *corev1.Secret) string { if secret == nil { - hash, _ := hash.ToHashString(hash.SHA256, "34Wd68DsJRUzrHp2f63o3U5hUD6zl8Tj") + hash, _ := hash2.ToHashString(hash2.SHA256, "34Wd68DsJRUzrHp2f63o3U5hUD6zl8Tj") return hash } - hash, _ := hash.ToHashString(hash.SHA256, secret.Data) + hash, _ := hash2.ToHashString(hash2.SHA256, secret.Data) return hash } diff --git a/pipeline-runner/steps/applyconfig/step.go b/pipeline-runner/steps/applyconfig/step.go index d1562e8d5..b2ba6c84e 100644 --- a/pipeline-runner/steps/applyconfig/step.go +++ b/pipeline-runner/steps/applyconfig/step.go @@ -21,6 +21,7 @@ import ( validate "github.com/equinor/radix-operator/pkg/apis/radixvalidators" operatorutils "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/equinor/radix-operator/pkg/apis/utils/git" + "github.com/equinor/radix-operator/pkg/apis/utils/hash" "github.com/rs/zerolog/log" corev1 "k8s.io/api/core/v1" kubeerrors "k8s.io/apimachinery/pkg/api/errors" @@ -401,7 +402,7 @@ func isRadixConfigNewOrModifiedSinceDeployment(ctx context.Context, rd *radixv1. if len(currentRdConfigHash) == 0 { return true, nil } - hashEqual, err := internal.CompareRadixApplicationHash(currentRdConfigHash, ra) + hashEqual, err := hash.CompareRadixApplicationHash(currentRdConfigHash, ra) if !hashEqual && err == nil { log.Ctx(ctx).Info().Msgf("RadixApplication updated since last deployment to environment %s", rd.Spec.Environment) } @@ -416,7 +417,7 @@ func isBuildSecretNewOrModifiedSinceDeployment(ctx context.Context, rd *radixv1. if len(targetHash) == 0 { return true, nil } - hashEqual, err := internal.CompareBuildSecretHash(targetHash, buildSecret) + hashEqual, err := hash.CompareBuildSecretHash(targetHash, buildSecret) if !hashEqual && err == nil { log.Ctx(ctx).Info().Msgf("Build secrets updated since last deployment to environment %s", rd.Spec.Environment) } diff --git a/pipeline-runner/steps/deploy/step.go b/pipeline-runner/steps/deploy/step.go index 4f30d9104..435249097 100644 --- a/pipeline-runner/steps/deploy/step.go +++ b/pipeline-runner/steps/deploy/step.go @@ -9,6 +9,7 @@ import ( "github.com/equinor/radix-operator/pipeline-runner/steps/internal" "github.com/equinor/radix-operator/pkg/apis/pipeline" "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/utils/hash" "github.com/rs/zerolog/log" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -76,12 +77,12 @@ func (cli *DeployStepImplementation) deployToEnv(ctx context.Context, appName, e commitID = pipelineInfo.PipelineArguments.CommitID // Commit ID specified by job arguments } - radixApplicationHash, err := internal.CreateRadixApplicationHash(pipelineInfo.RadixApplication) + radixApplicationHash, err := hash.CreateRadixApplicationHash(pipelineInfo.RadixApplication) if err != nil { return err } - buildSecretHash, err := internal.CreateBuildSecretHash(pipelineInfo.BuildSecret) + buildSecretHash, err := hash.CreateBuildSecretHash(pipelineInfo.BuildSecret) if err != nil { return err } diff --git a/pipeline-runner/steps/deployconfig/step_test.go b/pipeline-runner/steps/deployconfig/step_test.go index 8f0adfa87..dfa7d38e4 100644 --- a/pipeline-runner/steps/deployconfig/step_test.go +++ b/pipeline-runner/steps/deployconfig/step_test.go @@ -11,10 +11,10 @@ import ( "github.com/equinor/radix-operator/pipeline-runner/internal/watcher" "github.com/equinor/radix-operator/pipeline-runner/model" "github.com/equinor/radix-operator/pipeline-runner/steps/deployconfig" - "github.com/equinor/radix-operator/pipeline-runner/steps/internal" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/utils/hash" radixfake "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" "github.com/golang/mock/gomock" kedafake "github.com/kedacore/keda/v2/pkg/generated/clientset/versioned/fake" @@ -683,8 +683,8 @@ func (s *deployConfigTestSuite) createRadixDeployments(deploymentBuildersProps [ } func (s *deployConfigTestSuite) buildRadixDeployments(deploymentBuildersProps []radixDeploymentBuildersProps, ra *radixv1.RadixApplication, sourceEnvMap map[string]radixv1.RadixDeployment) []radixv1.RadixDeployment { - radixConfigHash, _ := internal.CreateRadixApplicationHash(ra) - buildSecretHash, _ := internal.CreateBuildSecretHash(nil) + radixConfigHash, _ := hash.CreateRadixApplicationHash(ra) + buildSecretHash, _ := hash.CreateBuildSecretHash(nil) var rdList []radixv1.RadixDeployment for _, rdProps := range deploymentBuildersProps { if sourceRd, ok := sourceEnvMap[rdProps.envName]; ok { diff --git a/pipeline-runner/steps/internal/hash.go b/pipeline-runner/steps/internal/hash.go deleted file mode 100644 index 071d0b05e..000000000 --- a/pipeline-runner/steps/internal/hash.go +++ /dev/null @@ -1,43 +0,0 @@ -package internal - -import ( - "github.com/equinor/radix-operator/pipeline-runner/internal/hash" - radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" - corev1 "k8s.io/api/core/v1" -) - -// Constants used to generate hash for RadixApplication and BuildSecret if they are nil. Do not change. -const ( - magicValueForNilRadixApplication = "0nXSg9l6EUepshGFmolpgV3elB0m8Mv7" - magicValueForNilBuildSecretData = "34Wd68DsJRUzrHp2f63o3U5hUD6zl8Tj" -) - -func CreateRadixApplicationHash(ra *radixv1.RadixApplication) (string, error) { - return hash.ToHashString(hash.SHA256, getRadixApplicationOrMagicValue(ra)) -} - -func CompareRadixApplicationHash(targetHash string, ra *radixv1.RadixApplication) (bool, error) { - return hash.CompareWithHashString(getRadixApplicationOrMagicValue(ra), targetHash) -} - -func CreateBuildSecretHash(secret *corev1.Secret) (string, error) { - return hash.ToHashString(hash.SHA256, getBuildSecretOrMagicValue(secret)) -} - -func CompareBuildSecretHash(targetHash string, secret *corev1.Secret) (bool, error) { - return hash.CompareWithHashString(getBuildSecretOrMagicValue(secret), targetHash) -} - -func getRadixApplicationOrMagicValue(ra *radixv1.RadixApplication) any { - if ra == nil { - return magicValueForNilRadixApplication - } - return ra.Spec -} - -func getBuildSecretOrMagicValue(secret *corev1.Secret) any { - if secret == nil || len(secret.Data) == 0 { - return magicValueForNilBuildSecretData - } - return secret.Data -} diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index 1d5109919..5e85afd88 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -583,29 +583,37 @@ func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appNa Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceStorage: requestsVolumeMountSize}, // it seems correct number is not needed for CSI driver }, - StorageClassName: &storageClassName, + VolumeName: pvcName, }, } return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) } -func populateCsiAzureStorageClass(storageClass *storagev1.StorageClass, appName string, volumeRootMount string, namespace string, componentName string, storageClassName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, provisioner string) error { - reclaimPolicy := corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimPolicy. PersistentVolumeReclaimPolicy deletes volume on unmount. +func populateCsiAzureVolumeMount(persistentVolume *corev1.PersistentVolume, appName string, volumeRootMount string, namespace string, componentName string, pvName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, provisioner string) error { bindingMode := getBindingMode(getRadixBlobFuse2VolumeMountBindingMode(radixVolumeMount)) - storageClass.ObjectMeta.Name = storageClassName - storageClass.ObjectMeta.Labels = getCsiAzureStorageClassLabels(appName, namespace, componentName, radixVolumeMount) - storageClass.Provisioner = provisioner - storageClass.Parameters = getCsiAzureStorageClassParameters(secretName, namespace, radixVolumeMount) + persistentVolume.ObjectMeta.Name = pvName + persistentVolume.ObjectMeta.Labels = getCsiAzurePersistentVolumeLabels(appName, namespace, componentName, radixVolumeMount) + persistentVolume.ObjectMeta.Annotations = getCsiAzurePersistentVolumeAnnotations(provisioner, secretName, namespace) + persistentVolume.Parameters = getCsiAzureStorageClassParameters(secretName, namespace, radixVolumeMount) mountOptions, err := getCsiAzureStorageClassMountOptions(volumeRootMount, namespace, componentName, radixVolumeMount) if err != nil { return err } - storageClass.MountOptions = mountOptions - storageClass.ReclaimPolicy = &reclaimPolicy - storageClass.VolumeBindingMode = &bindingMode + persistentVolume.Spec.MountOptions = mountOptions + persistentVolume.Spec.CSI.NodePublishSecretRef = &corev1.SecretReference{Name: secretName, Namespace: namespace} // TODO - remove for Workload identity + persistentVolume.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimPolicy. PersistentVolumeReclaimPolicy deletes volume on unmount. + persistentVolume.VolumeBindingMode = &bindingMode return nil } +func getCsiAzurePersistentVolumeAnnotations(provisioner, secretName, namespace string) map[string]string { + return map[string]string{ + "pv.kubernetes.io/provisioned-by: blob.csi.azure.com": provisioner, + "volume.kubernetes.io/provisioner-deletion-secret-name": secretName, + "volume.kubernetes.io/provisioner-deletion-secret-namespace": namespace, + } +} + func getBindingMode(bindingModeValue string) storagev1.VolumeBindingMode { if strings.EqualFold(strings.ToLower(bindingModeValue), strings.ToLower(string(storagev1.VolumeBindingWaitForFirstConsumer))) { return storagev1.VolumeBindingWaitForFirstConsumer @@ -613,7 +621,7 @@ func getBindingMode(bindingModeValue string) storagev1.VolumeBindingMode { return storagev1.VolumeBindingImmediate } -func getCsiAzureStorageClassLabels(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { +func getCsiAzurePersistentVolumeLabels(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { return map[string]string{ kube.RadixAppLabel: appName, kube.RadixNamespace: namespace, @@ -858,7 +866,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont return err } - scMap := utils.GetStorageClassMap(&scList.Items) + scMap := utils.GetPersistentVolumeMap(&scList.Items) pvcMap := utils.GetPersistentVolumeClaimMap(&pvcList.Items) radixVolumeMountMap := deploy.getRadixVolumeMountMapByCsiAzureVolumeMountName(componentName) var actualStorageClassNames []string @@ -874,22 +882,18 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont if !existsRadixVolumeMount { return fmt.Errorf("not found Radix volume mount for desired volume %s", volume.Name) } - storageClass, storageClassIsCreated, err := deploy.getOrCreateCsiAzureVolumeMountStorageClass(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, volume.Name, scMap) + pv, storageClassIsCreated, err := deploy.getOrCreateCsiAzurePersistentVolume(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, volume.Name, scMap) if err != nil { return err } - actualStorageClassNames = append(actualStorageClassNames, storageClass.Name) - pvc, err := deploy.createCsiAzurePersistentVolumeClaim(ctx, storageClass, storageClassIsCreated, appName, namespace, componentName, radixVolumeMount, volume.PersistentVolumeClaim.ClaimName, pvcMap) + actualStorageClassNames = append(actualStorageClassNames, pv.Name) + pvc, err := deploy.createCsiAzurePersistentVolumeClaim(ctx, pv, storageClassIsCreated, appName, namespace, componentName, radixVolumeMount, volume.PersistentVolumeClaim.ClaimName, pvcMap) if err != nil { return err } volume.PersistentVolumeClaim.ClaimName = pvc.Name actualPvcNames[pvc.Name] = struct{}{} } - err = deploy.garbageCollectCsiAzureStorageClasses(ctx, scList, actualStorageClassNames) - if err != nil { - return err - } err = deploy.garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx, namespace, pvcList, actualPvcNames) if err != nil { return err @@ -928,20 +932,6 @@ func addUsedPersistenceVolumeClaimsFrom(podTemplate corev1.PodTemplateSpec, pvcM } } -func (deploy *Deployment) garbageCollectCsiAzureStorageClasses(ctx context.Context, scList *storagev1.StorageClassList, excludeStorageClassName []string) error { - for _, storageClass := range scList.Items { - if commonUtils.ContainsString(excludeStorageClassName, storageClass.Name) { - continue - } - log.Ctx(ctx).Debug().Msgf("Delete Csi Azure StorageClass %s", storageClass.Name) - err := deploy.deleteCsiAzureStorageClasses(ctx, storageClass.Name) - if err != nil { - return err - } - } - return nil -} - func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx context.Context, namespace string, pvcList *corev1.PersistentVolumeClaimList, excludePvcNames map[string]any) error { for _, pvc := range pvcList.Items { if _, ok := excludePvcNames[pvc.Name]; ok { @@ -962,40 +952,39 @@ func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersist return nil } -func (deploy *Deployment) createCsiAzurePersistentVolumeClaim(ctx context.Context, storageClass *storagev1.StorageClass, requiredNewPvc bool, appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumeClaimName string, pvcMap map[string]*corev1.PersistentVolumeClaim) (*corev1.PersistentVolumeClaim, error) { +func (deploy *Deployment) createCsiAzurePersistentVolumeClaim(ctx context.Context, pv *corev1.PersistentVolume, requiredNewPvc bool, appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumeClaimName string, pvcMap map[string]*corev1.PersistentVolumeClaim) (*corev1.PersistentVolumeClaim, error) { if pvc, ok := pvcMap[persistentVolumeClaimName]; ok { - if pvc.Spec.StorageClassName == nil || len(*pvc.Spec.StorageClassName) == 0 { + if len(pvc.Spec.VolumeName) == 0 { return pvc, nil } - if !requiredNewPvc && strings.EqualFold(*pvc.Spec.StorageClassName, storageClass.Name) { + if !requiredNewPvc && strings.EqualFold(pvc.Spec.VolumeName, pv.Name) { return pvc, nil } - log.Ctx(ctx).Debug().Msgf("Delete in garbage-collect an old PersistentVolumeClaim %s in namespace %s: changed StorageClass name to %s", pvc.Name, namespace, storageClass.Name) + log.Ctx(ctx).Debug().Msgf("Delete in garbage-collect an old PersistentVolumeClaim %s in namespace %s: changed PersistentVolume name to %s", pvc.Name, namespace, pv.Name) } persistentVolumeClaimName, err := createCsiAzurePersistentVolumeClaimName(componentName, radixVolumeMount) if err != nil { return nil, err } - log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for StorageClass %s", persistentVolumeClaimName, namespace, storageClass.Name) - return deploy.createPersistentVolumeClaim(ctx, appName, namespace, componentName, persistentVolumeClaimName, storageClass.Name, radixVolumeMount) + log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for StorageClass %s", persistentVolumeClaimName, namespace, pv.Name) + return deploy.createPersistentVolumeClaim(ctx, appName, namespace, componentName, persistentVolumeClaimName, pv.Name, radixVolumeMount) } -// getOrCreateCsiAzureVolumeMountStorageClass returns creates or existing StorageClass, storageClassIsCreated=true, if created; error, if any -func (deploy *Deployment) getOrCreateCsiAzureVolumeMountStorageClass(ctx context.Context, appName, volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, volumeName string, scMap map[string]*storagev1.StorageClass) (*storagev1.StorageClass, bool, error) { +func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Context, appName, volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, volumeName string, pvMap map[string]*corev1.PersistentVolume) (*corev1.PersistentVolume, bool, error) { var volumeMountProvisioner, foundProvisioner = getStorageClassProvisionerByVolumeMountType(radixVolumeMount) if !foundProvisioner { return nil, false, fmt.Errorf("not found Storage Class provisioner for volume mount type %s", string(GetCsiAzureVolumeMountType(radixVolumeMount))) } - storageClassName := GetCsiAzureStorageClassName(namespace, volumeName) + pvName := GetCsiAzureStorageClassName(namespace, volumeName) csiVolumeSecretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, radixVolumeMount.Name) - if existingStorageClass, exists := scMap[storageClassName]; exists { + if existingStorageClass, exists := pvMap[pvName]; exists { desiredStorageClass := existingStorageClass.DeepCopy() - err := populateCsiAzureStorageClass(desiredStorageClass, appName, volumeRootMount, namespace, componentName, storageClassName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) + err := populateCsiAzureVolumeMount(desiredStorageClass, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) if err != nil { return nil, false, err } - if equal, err := utils.EqualStorageClasses(existingStorageClass, desiredStorageClass); equal || err != nil { + if equal, err := utils.EqualPersistentVolumes(existingStorageClass, desiredStorageClass); equal || err != nil { return existingStorageClass, false, err } @@ -1006,9 +995,9 @@ func (deploy *Deployment) getOrCreateCsiAzureVolumeMountStorageClass(ctx context } } - log.Ctx(ctx).Debug().Msgf("Create StorageClass %s in namespace %s", storageClassName, namespace) + log.Ctx(ctx).Debug().Msgf("Create StorageClass %s in namespace %s", pvName, namespace) storageClass := &storagev1.StorageClass{} - err := populateCsiAzureStorageClass(storageClass, appName, volumeRootMount, namespace, componentName, storageClassName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) + err := populateCsiAzureVolumeMount(storageClass, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) if err != nil { return nil, false, err } diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index a28d5c23a..78c1065a9 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -1182,7 +1182,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { equalPvcLists, err := utils.EqualPvcLists(&scenario.existingPvcsAfterTestRun, &existingPvcs, true) assert.Nil(t, err) assert.True(t, equalPvcLists) - equalStorageClassLists, err := utils.EqualStorageClassLists(&scenario.existingStorageClassesAfterTestRun, &existingScs) + equalStorageClassLists, err := utils.EqualPersistentVolumeLists(&scenario.existingStorageClassesAfterTestRun, &existingScs) assert.Nil(t, err) assert.True(t, equalStorageClassLists) } diff --git a/pipeline-runner/internal/hash/encoding.go b/pkg/apis/utils/hash/encoding.go similarity index 100% rename from pipeline-runner/internal/hash/encoding.go rename to pkg/apis/utils/hash/encoding.go diff --git a/pkg/apis/utils/hash/hash.go b/pkg/apis/utils/hash/hash.go new file mode 100644 index 000000000..96983a848 --- /dev/null +++ b/pkg/apis/utils/hash/hash.go @@ -0,0 +1,102 @@ +package hash + +import ( + "encoding/hex" + "fmt" + "strings" + + "github.com/equinor/radix-operator/pkg/apis/radix/v1" + v2 "k8s.io/api/core/v1" +) + +type Algorithm string + +const ( + hashStringSeparator string = "=" +) + +type sumFunc func(in []byte) []byte + +var sumProviders map[Algorithm]sumFunc = map[Algorithm]sumFunc{} + +func registerProvider(alg Algorithm, provider sumFunc) { + sumProviders[alg] = provider +} + +func ToHashString(alg Algorithm, source any) (string, error) { + sum, found := sumProviders[alg] + if !found { + return "", fmt.Errorf("hash algorithm %s not found", alg) + } + + encode, err := getEncoder(source) + if err != nil { + return "", err + } + + b, err := encode(source) + if err != nil { + return "", err + } + + hashBytes := sum(b) + return joinToHashString(alg, hex.EncodeToString(hashBytes)), nil +} + +func CompareWithHashString(source any, targetHashString string) (bool, error) { + alg := extractAlgorithmFromHashString(targetHashString) + sourceHashString, err := ToHashString(alg, source) + if err != nil { + return false, err + } + return targetHashString == sourceHashString, nil +} + +func joinToHashString(alg Algorithm, hash string) string { + return fmt.Sprintf("%s%s%s", alg, hashStringSeparator, hash) +} + +func extractAlgorithmFromHashString(hashString string) Algorithm { + parts := strings.Split(hashString, hashStringSeparator) + if len(parts) != 2 { + return "" + } + return Algorithm(parts[0]) +} + +// Constants used to generate hash for RadixApplication and BuildSecret if they are nil. Do not change. +const ( + magicValueForNilRadixApplication = "0nXSg9l6EUepshGFmolpgV3elB0m8Mv7" + magicValueForNilBuildSecretData = "34Wd68DsJRUzrHp2f63o3U5hUD6zl8Tj" + magicValueForNilVolumeMountData = "50gfDfg6h65fGfdVfs4r5dhnGFF4D35f" +) + +func CreateRadixApplicationHash(ra *v1.RadixApplication) (string, error) { + return ToHashString(SHA256, getRadixApplicationOrMagicValue(ra)) +} + +func CompareRadixApplicationHash(targetHash string, ra *v1.RadixApplication) (bool, error) { + return CompareWithHashString(getRadixApplicationOrMagicValue(ra), targetHash) +} + +func CreateBuildSecretHash(secret *v2.Secret) (string, error) { + return ToHashString(SHA256, getBuildSecretOrMagicValue(secret)) +} + +func CompareBuildSecretHash(targetHash string, secret *v2.Secret) (bool, error) { + return CompareWithHashString(getBuildSecretOrMagicValue(secret), targetHash) +} + +func getRadixApplicationOrMagicValue(ra *v1.RadixApplication) any { + if ra == nil { + return magicValueForNilRadixApplication + } + return ra.Spec +} + +func getBuildSecretOrMagicValue(secret *v2.Secret) any { + if secret == nil || len(secret.Data) == 0 { + return magicValueForNilBuildSecretData + } + return secret.Data +} diff --git a/pipeline-runner/internal/hash/hash_test.go b/pkg/apis/utils/hash/hash_test.go similarity index 92% rename from pipeline-runner/internal/hash/hash_test.go rename to pkg/apis/utils/hash/hash_test.go index 572e75c16..b21b18f9d 100644 --- a/pipeline-runner/internal/hash/hash_test.go +++ b/pkg/apis/utils/hash/hash_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/equinor/radix-operator/pipeline-runner/internal/hash" + hash2 "github.com/equinor/radix-operator/pkg/apis/utils/hash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,10 +16,10 @@ func Test_ToHashString(t *testing.T) { } type algorithm struct { - alg hash.Algorithm + alg hash2.Algorithm hashLen int } - algorithms := []algorithm{{alg: hash.SHA256, hashLen: 32}} + algorithms := []algorithm{{alg: hash2.SHA256, hashLen: 32}} type testSpec struct { test string @@ -65,14 +65,14 @@ func Test_ToHashString(t *testing.T) { for _, test := range tests { for _, alg := range algorithms { t.Run(test.test, func(t *testing.T) { - valHash, err := hash.ToHashString(alg.alg, test.val) + valHash, err := hash2.ToHashString(alg.alg, test.val) require.NoError(t, err) hashParts := strings.Split(valHash, "=") assert.Equal(t, string(alg.alg), hashParts[0]) valHashBytes, err := hex.DecodeString(hashParts[1]) require.NoError(t, err) assert.Len(t, valHashBytes, alg.hashLen) - match, err := hash.CompareWithHashString(test.valOther, valHash) + match, err := hash2.CompareWithHashString(test.valOther, valHash) require.NoError(t, err) assert.Equal(t, test.match, match) t.Parallel() diff --git a/pipeline-runner/internal/hash/sha256.go b/pkg/apis/utils/hash/sha256.go similarity index 100% rename from pipeline-runner/internal/hash/sha256.go rename to pkg/apis/utils/hash/sha256.go diff --git a/pkg/apis/utils/storageclass.go b/pkg/apis/utils/storageclass.go index 3886e77ca..7f6ef12cf 100644 --- a/pkg/apis/utils/storageclass.go +++ b/pkg/apis/utils/storageclass.go @@ -3,86 +3,87 @@ package utils import ( "encoding/json" "fmt" + "github.com/equinor/radix-operator/pkg/apis/kube" - storagev1 "k8s.io/api/storage/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/strategicpatch" ) -// GetStorageClassMap Get map from StorageClassList with name as key -func GetStorageClassMap(scList *[]storagev1.StorageClass) map[string]*storagev1.StorageClass { - scMap := make(map[string]*storagev1.StorageClass) - for _, sc := range *scList { - sc := sc - scMap[sc.Name] = &sc +// GetPersistentVolumeMap Get map from PersistentVolumeList with name as key +func GetPersistentVolumeMap(pvList *[]corev1.PersistentVolume) map[string]*corev1.PersistentVolume { + pvMap := make(map[string]*corev1.PersistentVolume) + for _, pv := range *pvList { + pv := pv + pvMap[pv.Name] = &pv } - return scMap + return pvMap } -// EqualStorageClassLists Compare two StorageClass lists -func EqualStorageClassLists(scList1, scList2 *[]storagev1.StorageClass) (bool, error) { - if len(*scList1) != len(*scList2) { - return false, fmt.Errorf("different StorageClass list sizes: %v, %v", len(*scList1), len(*scList2)) +// EqualPersistentVolumeLists Compare two PersistentVolume lists +func EqualPersistentVolumeLists(list1, list2 *[]corev1.PersistentVolume) (bool, error) { + if len(*list1) != len(*list2) { + return false, fmt.Errorf("different PersistentVolume list sizes: %v, %v", len(*list1), len(*list2)) } - scMap1 := GetStorageClassMap(scList1) - scMap2 := GetStorageClassMap(scList2) - for scName, sc1 := range scMap1 { - sc2, ok := scMap2[scName] + map1 := GetPersistentVolumeMap(list1) + map2 := GetPersistentVolumeMap(list2) + for pvName, pv1 := range map1 { + pv2, ok := map2[pvName] if !ok { - return false, fmt.Errorf("StorageClass not found by name %s in second list", scName) + return false, fmt.Errorf("PersistentVolume not found by name %s in second list", pvName) } - if equal, err := EqualStorageClasses(sc1, sc2); err != nil || !equal { + if equal, err := EqualPersistentVolumes(pv1, pv2); err != nil || !equal { return false, err } } return true, nil } -// EqualStorageClasses Compare two StorageClass pointers -func EqualStorageClasses(sc1, sc2 *storagev1.StorageClass) (bool, error) { - sc1Copy, labels1, params1, mountOptions1 := getStorageClassCopyWithCollections(sc1) - sc2Copy, labels2, params2, mountOptions2 := getStorageClassCopyWithCollections(sc2) - patchBytes, err := getStorageClassesPatch(sc1Copy, sc2Copy) +// EqualPersistentVolumes Compare two PersistentVolumes +func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) (bool, error) { + pv1Copy, labels1, params1, mountOptions1 := getPersistentVolumeCopyWithCollections(pv1) + pv2Copy, labels2, params2, mountOptions2 := getPersistentVolumeCopyWithCollections(pv2) + patchBytes, err := getPersistentVolumePatch(pv1Copy, pv2Copy) if err != nil { return false, err } if !EqualStringMaps(labels1, labels2) { - return false, nil //StorageClasses labels are not equal + return false, nil // StorageClasses labels are not equal } if !EqualStringMaps(params1, params2) { - return false, nil //StorageClasses parameters are not equal + return false, nil // StorageClasses parameters are not equal } if !EqualStringLists(mountOptions1, mountOptions2) { - return false, nil //StorageClass-es MountOptions are not equal + return false, nil // StorageClass-es MountOptions are not equal } if !kube.IsEmptyPatch(patchBytes) { - return false, nil //StorageClasses properties are not equal + return false, nil // StorageClasses properties are not equal } return true, nil } -func getStorageClassCopyWithCollections(sc *storagev1.StorageClass) (*storagev1.StorageClass, map[string]string, map[string]string, []string) { - scCopy := sc.DeepCopy() - scCopy.ObjectMeta.ManagedFields = nil //HACK: to avoid ManagedFields comparison - //to avoid label order variations - labels := scCopy.ObjectMeta.Labels - scCopy.ObjectMeta.Labels = map[string]string{} - //to avoid Parameters order variations - scParams := scCopy.Parameters - scCopy.Parameters = map[string]string{} - //to avoid MountOptions order variations - scMountOptions := scCopy.MountOptions - scCopy.MountOptions = []string{} - return scCopy, labels, scParams, scMountOptions +func getPersistentVolumeCopyWithCollections(pv *corev1.PersistentVolume) (*corev1.PersistentVolume, map[string]string, map[string]string, []string) { + pvCopy := pv.DeepCopy() + pvCopy.ObjectMeta.ManagedFields = nil // HACK: to avoid ManagedFields comparison + // to avoid label order variations + labels := pvCopy.ObjectMeta.Labels + pvCopy.ObjectMeta.Labels = map[string]string{} + // to avoid Parameters order variations + scParams := pvCopy.Spec.Parameters // TODO nodeStorageSecretRef: name, ns + pvCopy.Parameters = map[string]string{} + // to avoid MountOptions order variations + mountOptions := pvCopy.Spec.MountOptions + pvCopy.Spec.MountOptions = []string{} + return pvCopy, labels, scParams, mountOptions } -func getStorageClassesPatch(sc1, sc2 *storagev1.StorageClass) ([]byte, error) { - json1, err := json.Marshal(sc1) +func getPersistentVolumePatch(pv1, pv2 *corev1.PersistentVolume) ([]byte, error) { + json1, err := json.Marshal(pv1) if err != nil { return []byte{}, err } - json2, err := json.Marshal(sc2) + json2, err := json.Marshal(pv2) if err != nil { return []byte{}, err } - return strategicpatch.CreateTwoWayMergePatch(json1, json2, storagev1.StorageClass{}) + return strategicpatch.CreateTwoWayMergePatch(json1, json2, corev1.PersistentVolume{}) } From 9c9d98fdd9abf31e35d6c6d7a7947896bb1252b7 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Wed, 13 Nov 2024 15:28:48 +0100 Subject: [PATCH 02/68] Replacing the storage class with pv --- pipeline-runner/internal/jobs/build/acr.go | 2 +- pkg/apis/batch/syncer_test.go | 17 +- pkg/apis/deployment/deployment.go | 2 +- pkg/apis/deployment/deployment_test.go | 8 +- pkg/apis/deployment/secrets.go | 2 +- pkg/apis/deployment/volumemount.go | 276 +++++------ pkg/apis/deployment/volumemount_test.go | 453 ++++-------------- pkg/apis/metrics/custom_metrics.go | 5 +- pkg/apis/radix/v1/radixapptypes.go | 46 +- pkg/apis/radixvalidators/errors.go | 4 - pkg/apis/radixvalidators/validate_ra.go | 16 +- pkg/apis/radixvalidators/validate_ra_test.go | 35 -- .../{storageclass.go => persistentvolume.go} | 22 +- 13 files changed, 260 insertions(+), 628 deletions(-) rename pkg/apis/utils/{storageclass.go => persistentvolume.go} (77%) diff --git a/pipeline-runner/internal/jobs/build/acr.go b/pipeline-runner/internal/jobs/build/acr.go index fc7db5a06..fc716cee5 100644 --- a/pipeline-runner/internal/jobs/build/acr.go +++ b/pipeline-runner/internal/jobs/build/acr.go @@ -34,7 +34,7 @@ func NewACR() JobsBuilder { type acr struct{} -func (c *acr) BuildJobs(useBuildCache bool, pipelineArgs model.PipelineArguments, cloneURL, gitCommitHash, gitTags string, componentImages []pipeline.BuildComponentImage, buildSecrets []string) []batchv1.Job { +func (c *acr) BuildJobs(_ bool, pipelineArgs model.PipelineArguments, cloneURL, gitCommitHash, gitTags string, componentImages []pipeline.BuildComponentImage, buildSecrets []string) []batchv1.Job { props := &acrKubeJobProps{ pipelineArgs: pipelineArgs, componentImages: componentImages, diff --git a/pkg/apis/batch/syncer_test.go b/pkg/apis/batch/syncer_test.go index 1c0ae2e29..f299b5078 100644 --- a/pkg/apis/batch/syncer_test.go +++ b/pkg/apis/batch/syncer_test.go @@ -1072,8 +1072,6 @@ func (s *syncerTestSuite) Test_JobWithVolumeMounts() { Name: componentName, VolumeMounts: []radixv1.RadixVolumeMount{ {Name: "azureblob2name", Path: "/azureblob2path", BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{Protocol: radixv1.BlobFuse2ProtocolFuse2, Container: "azureblob2container"}}, - {Name: "azurenfsname", Path: "/azurenfspath", BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{Protocol: radixv1.BlobFuse2ProtocolNfs, Container: "azurenfscontainer"}}, - {Name: "azurefilename", Path: "/azurefilepath", AzureFile: &radixv1.RadixAzureFileVolumeMount{Share: "azurefilecontainer"}}, }, }, }, @@ -1089,14 +1087,10 @@ func (s *syncerTestSuite) Test_JobWithVolumeMounts() { jobs, _ := s.kubeClient.BatchV1().Jobs(namespace).List(context.Background(), metav1.ListOptions{}) s.Require().Len(jobs.Items, 1) job := slice.FindAll(jobs.Items, func(job batchv1.Job) bool { return job.GetName() == getKubeJobName(batchName, jobName) })[0] - s.Require().Len(job.Spec.Template.Spec.Volumes, 3) - s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 3) + s.Require().Len(job.Spec.Template.Spec.Volumes, 2) + s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 2) s.Equal(job.Spec.Template.Spec.Volumes[0].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name) - s.Equal(job.Spec.Template.Spec.Volumes[1].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[1].Name) - s.Equal(job.Spec.Template.Spec.Volumes[2].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[2].Name) s.Equal("/azureblob2path", job.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath) - s.Equal("/azurenfspath", job.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath) - s.Equal("/azurefilepath", job.Spec.Template.Spec.Containers[0].VolumeMounts[2].MountPath) } func (s *syncerTestSuite) Test_JobWithVolumeMounts_Deprecated() { @@ -1122,7 +1116,6 @@ func (s *syncerTestSuite) Test_JobWithVolumeMounts_Deprecated() { VolumeMounts: []radixv1.RadixVolumeMount{ {Type: "blob", Name: "blobname", Container: "blobcontainer", Path: "/blobpath"}, {Type: "azure-blob", Name: "azureblobname", Storage: "azureblobcontainer", Path: "/azureblobpath"}, - {Type: "azure-file", Name: "azurefilename", Storage: "azurefilecontainer", Path: "/azurefilepath"}, }, }, }, @@ -1138,14 +1131,12 @@ func (s *syncerTestSuite) Test_JobWithVolumeMounts_Deprecated() { jobs, _ := s.kubeClient.BatchV1().Jobs(namespace).List(context.Background(), metav1.ListOptions{}) s.Require().Len(jobs.Items, 1) job := slice.FindAll(jobs.Items, func(job batchv1.Job) bool { return job.GetName() == getKubeJobName(batchName, jobName) })[0] - s.Require().Len(job.Spec.Template.Spec.Volumes, 3) - s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 3) + s.Require().Len(job.Spec.Template.Spec.Volumes, 2) + s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 2) s.Equal(job.Spec.Template.Spec.Volumes[0].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name) s.Equal(job.Spec.Template.Spec.Volumes[1].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[1].Name) - s.Equal(job.Spec.Template.Spec.Volumes[2].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[2].Name) s.Equal("/blobpath", job.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath) s.Equal("/azureblobpath", job.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath) - s.Equal("/azurefilepath", job.Spec.Template.Spec.Containers[0].VolumeMounts[2].MountPath) } func (s *syncerTestSuite) Test_JobWithAzureSecretRefs() { diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 8d40130da..e7803bae5 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -480,7 +480,7 @@ func getLabelSelectorForBlobVolumeMountSecret(component v1.RadixCommonDeployComp } func getLabelSelectorForCsiAzureVolumeMountSecret(component v1.RadixCommonDeployComponent) string { - return fmt.Sprintf("%s=%s, %s in (%s, %s, %s, %s)", kube.RadixComponentLabel, component.GetName(), kube.RadixMountTypeLabel, string(v1.MountTypeBlobFuse2FuseCsiAzure), string(v1.MountTypeBlobFuse2Fuse2CsiAzure), string(v1.MountTypeBlobFuse2NfsCsiAzure), string(v1.MountTypeAzureFileCsiAzure)) + return fmt.Sprintf("%s=%s, %s in (%s, %s, %s, %s)", kube.RadixComponentLabel, component.GetName(), kube.RadixMountTypeLabel, string(v1.MountTypeBlobFuse2FuseCsiAzure), string(v1.MountTypeBlobFuse2Fuse2CsiAzure)) } func (deploy *Deployment) maintainHistoryLimit(ctx context.Context, deploymentHistoryLimit int) { diff --git a/pkg/apis/deployment/deployment_test.go b/pkg/apis/deployment/deployment_test.go index ac4b301a5..e1ad68b1d 100644 --- a/pkg/apis/deployment/deployment_test.go +++ b/pkg/apis/deployment/deployment_test.go @@ -3860,7 +3860,6 @@ func Test_ComponentSynced_VolumeAndMounts(t *testing.T) { WithVolumeMounts( radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlob, Name: "blob", Container: "blobcontainer", Path: "blobpath"}, radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blobcsi", Storage: "blobcsistorage", Path: "blobcsipath"}, - radixv1.RadixVolumeMount{Type: radixv1.MountTypeAzureFileCsiAzure, Name: "filecsi", Storage: "filecsistorage", Path: "filecsipath"}, ), ), ) @@ -3869,8 +3868,8 @@ func Test_ComponentSynced_VolumeAndMounts(t *testing.T) { envNamespace := utils.GetEnvironmentNamespace(appName, environment) deployment, _ := client.AppsV1().Deployments(envNamespace).Get(context.Background(), compName, metav1.GetOptions{}) require.NotNil(t, deployment) - assert.Len(t, deployment.Spec.Template.Spec.Volumes, 3, "incorrect number of volumes") - assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 3, "incorrect number of volumemounts") + assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2, "incorrect number of volumes") + assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 2, "incorrect number of volumemounts") } func Test_JobSynced_VolumeAndMounts(t *testing.T) { @@ -3893,7 +3892,6 @@ func Test_JobSynced_VolumeAndMounts(t *testing.T) { WithVolumeMounts( radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlob, Name: "blob", Container: "blobcontainer", Path: "blobpath"}, radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blobcsi", Storage: "blobcsistorage", Path: "blobcsipath"}, - radixv1.RadixVolumeMount{Type: radixv1.MountTypeAzureFileCsiAzure, Name: "filecsi", Storage: "filecsistorage", Path: "filecsipath"}, ), ), ) @@ -3903,7 +3901,7 @@ func Test_JobSynced_VolumeAndMounts(t *testing.T) { deploymentList, _ := client.AppsV1().Deployments(envNamespace).List(context.Background(), metav1.ListOptions{LabelSelector: radixlabels.ForJobAuxObject(jobName, kube.RadixJobTypeManagerAux).String()}) require.Len(t, deploymentList.Items, 1) deployment := deploymentList.Items[0] - assert.Len(t, deployment.Spec.Template.Spec.Volumes, 3, "incorrect number of volumes") + assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2, "incorrect number of volumes") assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 0, "incorrect number of volumemounts") } diff --git a/pkg/apis/deployment/secrets.go b/pkg/apis/deployment/secrets.go index 926d96584..cba9a881b 100644 --- a/pkg/apis/deployment/secrets.go +++ b/pkg/apis/deployment/secrets.go @@ -126,7 +126,7 @@ func (deploy *Deployment) createOrUpdateVolumeMountSecrets(ctx context.Context, return nil, err } } - case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeBlobFuse2Fuse2CsiAzure, radixv1.MountTypeBlobFuse2NfsCsiAzure, radixv1.MountTypeAzureFileCsiAzure: + case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeBlobFuse2Fuse2CsiAzure: { secretName, accountKey, accountName := deploy.getCsiAzureVolumeMountCredsSecrets(ctx, namespace, componentName, volumeMount.Name) volumeMountSecretsToManage = append(volumeMountSecretsToManage, secretName) diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index 5e85afd88..7be5e683d 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -14,7 +14,6 @@ import ( "github.com/rs/zerolog/log" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,30 +31,30 @@ const ( csiVolumeNameTemplate = "%s-%s-%s-%s" // --- csiPersistentVolumeClaimNameTemplate = "pvc-%s-%s" // pvc-- - csiStorageClassNameTemplate = "sc-%s-%s" // sc-- + csiPersistentVolumeNameTemplate = "sc-%s-%s" // sc-- csiVolumeNodeMountPathTemplate = "%s/%s/%s/%s/%s/%s" // ///// - csiStorageClassProvisionerSecretNameParameter = "csi.storage.k8s.io/provisioner-secret-name" // Secret name, containing storage account name and key - csiStorageClassProvisionerSecretNamespaceParameter = "csi.storage.k8s.io/provisioner-secret-namespace" // namespace of the secret - csiStorageClassNodeStageSecretNameParameter = "csi.storage.k8s.io/node-stage-secret-name" // Usually equal to csiStorageClassProvisionerSecretNameParameter - csiStorageClassNodeStageSecretNamespaceParameter = "csi.storage.k8s.io/node-stage-secret-namespace" // Usually equal to csiStorageClassProvisionerSecretNamespaceParameter - csiAzureStorageClassSkuNameParameter = "skuName" // Available values: Standard_LRS (default), Premium_LRS, Standard_GRS, Standard_RAGRS. https://docs.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - csiStorageClassContainerNameParameter = "containerName" // Container name - foc container storages - csiStorageClassShareNameParameter = "shareName" // File Share name - for file storages - csiStorageClassTmpPathMountOption = "tmp-path" // Path within the node, where the volume mount has been mounted to - csiStorageClassGidMountOption = "gid" // Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting - csiStorageClassUidMountOption = "uid" // Volume mount owner UserID. Used instead of GroupID - csiStorageClassUseAdlsMountOption = "use-adls" // Use ADLS or Block Blob - csiStorageClassStreamingEnabledMountOption = "streaming" // Enable Streaming - csiStorageClassStreamingCacheMountOption = "stream-cache-mb" // Limit total amount of data being cached in memory to conserve memory - csiStorageClassStreamingMaxBlocksPerFileMountOption = "max-blocks-per-file" // Maximum number of blocks to be cached in memory for streaming - csiStorageClassStreamingMaxBuffersMountOption = "max-buffers" // The total number of buffers to be cached in memory (in MB). - csiStorageClassStreamingBlockSizeMountOption = "block-size-mb" // The size of each block to be cached in memory (in MB). - csiStorageClassStreamingBufferSizeMountOption = "buffer-size-mb" // The size of each buffer to be cached in memory (in MB). - csiStorageClassProtocolParameter = "protocol" // Protocol - csiStorageClassProtocolParameterFuse = "fuse" // Protocol "blobfuse" - csiStorageClassProtocolParameterFuse2 = "fuse2" // Protocol "blobfuse2" - csiStorageClassProtocolParameterNfs = "nfs" // Protocol "nfs" + csiPersistentVolumeProvisionerSecretNameParameter = "csi.storage.k8s.io/provisioner-secret-name" // Secret name, containing storage account name and key + csiPersistentVolumeProvisionerSecretNamespaceParameter = "csi.storage.k8s.io/provisioner-secret-namespace" // namespace of the secret + csiPersistentVolumeNodeStageSecretNameParameter = "csi.storage.k8s.io/node-stage-secret-name" // Usually equal to csiPersistentVolumeProvisionerSecretNameParameter + csiPersistentVolumeNodeStageSecretNamespaceParameter = "csi.storage.k8s.io/node-stage-secret-namespace" // Usually equal to csiPersistentVolumeProvisionerSecretNamespaceParameter + csiVolumeMountAttributeContainerName = "containerName" // Container name - foc container storages + csiPersistentVolumeTmpPathMountOption = "tmp-path" // Path within the node, where the volume mount has been mounted to + csiPersistentVolumeGidMountOption = "gid" // Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting + csiPersistentVolumeUidMountOption = "uid" // Volume mount owner UserID. Used instead of GroupID + csiPersistentVolumeUseAdlsMountOption = "use-adls" // Use ADLS or Block Blob + csiPersistentVolumeStreamingEnabledMountOption = "streaming" // Enable Streaming + csiPersistentVolumeStreamingCacheMountOption = "stream-cache-mb" // Limit total amount of data being cached in memory to conserve memory + csiPersistentVolumeStreamingMaxBlocksPerFileMountOption = "max-blocks-per-file" // Maximum number of blocks to be cached in memory for streaming + csiPersistentVolumeStreamingMaxBuffersMountOption = "max-buffers" // The total number of buffers to be cached in memory (in MB). + csiPersistentVolumeStreamingBlockSizeMountOption = "block-size-mb" // The size of each block to be cached in memory (in MB). + csiPersistentVolumeStreamingBufferSizeMountOption = "buffer-size-mb" // The size of each buffer to be cached in memory (in MB). + csiVolumeMountAttributeProtocol = "protocol" // Protocol + csiPersistentVolumeProtocolParameterFuse = "fuse" // Protocol "blobfuse" + csiPersistentVolumeProtocolParameterFuse2 = "fuse2" // Protocol "blobfuse2" + csiVolumeMountAttributeStorageAccount = "storageAccount" + csiVolumeMountAttributeClientID = "clientID" + csiVolumeMountAttributeResourceGroup = "resourcegroup" csiSecretStoreDriver = "secrets-store.csi.k8s.io" csiVolumeSourceVolumeAttrSecretProviderClassName = "secretProviderClass" @@ -68,27 +67,20 @@ const ( const ( // provisionerBlobCsiAzure Use of azure/csi driver for blob in Azure storage account provisionerBlobCsiAzure string = "blob.csi.azure.com" - // provisionerFileCsiAzure Use of azure/csi driver for files in Azure storage account - provisionerFileCsiAzure string = "file.csi.azure.com" ) var ( - csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}, provisionerFileCsiAzure: struct{}{}} + csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}} ) -// getStorageClassProvisionerByVolumeMountType convert volume mount type to Storage Class provisioner -func getStorageClassProvisionerByVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) (string, bool) { +// getPersistentVolumeProvisionerByVolumeMountType convert volume mount type to Storage Class provisioner +func getPersistentVolumeProvisionerByVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) (string, bool) { if radixVolumeMount.BlobFuse2 != nil { return provisionerBlobCsiAzure, true } - if radixVolumeMount.AzureFile != nil { - return provisionerFileCsiAzure, true - } switch radixVolumeMount.Type { - case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeBlobFuse2Fuse2CsiAzure, radixv1.MountTypeBlobFuse2NfsCsiAzure: + case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeBlobFuse2Fuse2CsiAzure: return provisionerBlobCsiAzure, true - case radixv1.MountTypeAzureFileCsiAzure: - return provisionerFileCsiAzure, true } return "", false } @@ -96,7 +88,7 @@ func getStorageClassProvisionerByVolumeMountType(radixVolumeMount *radixv1.Radix // isKnownCsiAzureVolumeMount Supported volume mount type CSI Azure Blob volume func isKnownCsiAzureVolumeMount(volumeMount string) bool { switch volumeMount { - case string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure), string(radixv1.MountTypeBlobFuse2NfsCsiAzure), string(radixv1.MountTypeAzureFileCsiAzure): + case string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure): return true } return false @@ -174,7 +166,7 @@ func getCsiAzureVolumeMountName(volumeMount *radixv1.RadixVolumeMount, component if len(volumeMount.Name) == 0 { return "", fmt.Errorf("name is empty for volume mount in the component %s", componentName) } - csiAzureVolumeStorageName := GetRadixVolumeMountStorage(volumeMount) + csiAzureVolumeStorageName := getRadixVolumeMountStorage(volumeMount) if len(csiAzureVolumeStorageName) == 0 { return "", fmt.Errorf("storage is empty for volume mount %s in the component %s", volumeMount.Name, componentName) } @@ -186,41 +178,27 @@ func getCsiAzureVolumeMountName(volumeMount *radixv1.RadixVolumeMount, component // GetCsiAzureVolumeMountType Gets the CSI Azure volume mount type func GetCsiAzureVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) radixv1.MountType { - if radixVolumeMount.BlobFuse2 != nil { - switch radixVolumeMount.BlobFuse2.Protocol { - case radixv1.BlobFuse2ProtocolFuse2, "": // default protocol if not set - return radixv1.MountTypeBlobFuse2Fuse2CsiAzure - case radixv1.BlobFuse2ProtocolNfs: - return radixv1.MountTypeBlobFuse2NfsCsiAzure - default: - return "unsupported" - } + if radixVolumeMount.BlobFuse2 == nil { + return radixVolumeMount.Type } - if radixVolumeMount.AzureFile != nil { - return radixv1.MountTypeAzureFileCsiAzure + switch radixVolumeMount.BlobFuse2.Protocol { + case radixv1.BlobFuse2ProtocolFuse2, "": // default protocol if not set + return radixv1.MountTypeBlobFuse2Fuse2CsiAzure + default: + return "unsupported" } - return radixVolumeMount.Type } func getCsiRadixVolumeTypeIdForName(radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { if radixVolumeMount.BlobFuse2 != nil { switch radixVolumeMount.BlobFuse2.Protocol { - case radixv1.BlobFuse2ProtocolFuse2, "": - return "csi-blobfuse2-fuse2", nil - case radixv1.BlobFuse2ProtocolNfs: - return "csi-blobfuse2-nfs", nil default: return "", fmt.Errorf("unknown blobfuse2 protocol %s", radixVolumeMount.BlobFuse2.Protocol) } } - if radixVolumeMount.AzureFile != nil { - return "csi-az-file", nil - } switch radixVolumeMount.Type { case radixv1.MountTypeBlobFuse2FuseCsiAzure: return "csi-az-blob", nil - case radixv1.MountTypeAzureFileCsiAzure: - return "csi-az-file", nil } return "", fmt.Errorf("unknown volume mount type %s", radixVolumeMount.Type) } @@ -310,8 +288,6 @@ func getComponentVolumeMountVolumes(ctx context.Context, kubeclient kubernetes.I return getComponentVolumeMountDeprecatedVolumeSource(ctx, volumeMount, namespace, environment, deployComponent.GetName(), kubeclient) case volumeMount.HasBlobFuse2(): return getComponentVolumeMountBlobFuse2VolumeSource(ctx, volumeMount, namespace, deployComponent.GetName(), kubeclient) - case volumeMount.HasAzureFile(): - return getComponentVolumeMountAzureFileVolumeSource(ctx, volumeMount, namespace, deployComponent.GetName(), kubeclient) case volumeMount.HasEmptyDir(): return getComponentVolumeMountEmptyDirVolumeSource(volumeMount.EmptyDir), nil } @@ -339,7 +315,7 @@ func getComponentVolumeMountDeprecatedVolumeSource(ctx context.Context, volumeMo switch volumeMount.Type { case radixv1.MountTypeBlob: return getBlobFuseVolume(namespace, environment, componentName, volumeMount), nil - case radixv1.MountTypeAzureFileCsiAzure, radixv1.MountTypeBlobFuse2FuseCsiAzure: + case radixv1.MountTypeBlobFuse2FuseCsiAzure: return getCsiAzureVolume(ctx, kubeclient, namespace, componentName, volumeMount) } @@ -350,10 +326,6 @@ func getComponentVolumeMountBlobFuse2VolumeSource(ctx context.Context, volumeMou return getCsiAzureVolume(ctx, kubeclient, namespace, componentName, volumeMount) } -func getComponentVolumeMountAzureFileVolumeSource(ctx context.Context, volumeMount *radixv1.RadixVolumeMount, namespace, componentName string, kubeclient kubernetes.Interface) (*corev1.VolumeSource, error) { - return getCsiAzureVolume(ctx, kubeclient, namespace, componentName, volumeMount) -} - func getComponentVolumeMountEmptyDirVolumeSource(spec *radixv1.RadixEmptyDirVolumeMount) *corev1.VolumeSource { return &corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{ @@ -390,17 +362,11 @@ func getVolumeMountVolumeName(volumeMount *radixv1.RadixVolumeMount, componentNa return getVolumeMountDeprecatedVolumeName(volumeMount, componentName) case volumeMount.HasBlobFuse2(): return getVolumeMountBlobFuse2VolumeName(volumeMount, componentName) - case volumeMount.HasAzureFile(): - return getVolumeMountAzureFileVolumeName(volumeMount, componentName) } return fmt.Sprintf("radix-vm-%s", volumeMount.Name), nil } -func getVolumeMountAzureFileVolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { - return getCsiAzureVolumeMountName(volumeMount, componentName) -} - func getVolumeMountBlobFuse2VolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { return getCsiAzureVolumeMountName(volumeMount, componentName) } @@ -409,7 +375,7 @@ func getVolumeMountDeprecatedVolumeName(volumeMount *radixv1.RadixVolumeMount, c switch volumeMount.Type { case radixv1.MountTypeBlob: return getBlobFuseVolumeMountName(volumeMount, componentName), nil - case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeAzureFileCsiAzure: + case radixv1.MountTypeBlobFuse2FuseCsiAzure: return getCsiAzureVolumeMountName(volumeMount, componentName) } @@ -444,9 +410,9 @@ func createCsiAzurePersistentVolumeClaimName(componentName string, radixVolumeMo return fmt.Sprintf(csiPersistentVolumeClaimNameTemplate, volumeName, strings.ToLower(commonUtils.RandString(5))), nil // volumeName: --- } -// GetCsiAzureStorageClassName hold a name of CSI volume storage class -func GetCsiAzureStorageClassName(namespace, volumeName string) string { - return fmt.Sprintf(csiStorageClassNameTemplate, namespace, volumeName) // volumeName: --- +// GetCsiAzurePersistentVolumeName hold a name of CSI volume storage class +func GetCsiAzurePersistentVolumeName(namespace, volumeName string) string { + return fmt.Sprintf(csiPersistentVolumeNameTemplate, namespace, volumeName) // volumeName: --- } func getBlobFuseVolume(namespace, environment, componentName string, volumeMount *radixv1.RadixVolumeMount) *corev1.VolumeSource { @@ -533,9 +499,9 @@ func (deploy *Deployment) garbageCollectVolumeMountsSecretsNoLongerInSpecForComp return deploy.GarbageCollectSecrets(ctx, secrets, excludeSecretNames) } -func (deploy *Deployment) getCsiAzureStorageClasses(ctx context.Context, namespace, componentName string) (*storagev1.StorageClassList, error) { - return deploy.kubeclient.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{ - LabelSelector: getLabelSelectorForCsiAzureStorageClass(namespace, componentName), +func (deploy *Deployment) getCsiAzurePersistentVolume(ctx context.Context, namespace, componentName string) (*corev1.PersistentVolumeList, error) { + return deploy.kubeclient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{ + LabelSelector: getLabelSelectorForCsiAzurePersistentVolume(namespace, componentName), }) } @@ -549,19 +515,19 @@ func (deploy *Deployment) getPersistentVolumesForPvc(ctx context.Context) (*core return deploy.kubeclient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) } -func getLabelSelectorForCsiAzureStorageClass(namespace, componentName string) string { - return fmt.Sprintf("%s=%s, %s=%s, %s in (%s, %s, %s, %s)", kube.RadixNamespace, namespace, kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure), string(radixv1.MountTypeBlobFuse2NfsCsiAzure), string(radixv1.MountTypeAzureFileCsiAzure)) +func getLabelSelectorForCsiAzurePersistentVolume(namespace, componentName string) string { + return fmt.Sprintf("%s=%s, %s=%s, %s in (%s, %s)", kube.RadixNamespace, namespace, kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure)) } func getLabelSelectorForCsiAzurePersistenceVolumeClaim(componentName string) string { - return fmt.Sprintf("%s=%s, %s in (%s, %s, %s, %s)", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure), string(radixv1.MountTypeBlobFuse2NfsCsiAzure), string(radixv1.MountTypeAzureFileCsiAzure)) + return fmt.Sprintf("%s=%s, %s in (%s, %s)", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure)) } func getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentStorage(componentName, radixVolumeMountName string) string { - return fmt.Sprintf("%s=%s, %s in (%s, %s, %s, %s), %s = %s", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure), string(radixv1.MountTypeBlobFuse2NfsCsiAzure), string(radixv1.MountTypeAzureFileCsiAzure), kube.RadixVolumeMountNameLabel, radixVolumeMountName) + return fmt.Sprintf("%s=%s, %s in (%s, %s), %s = %s", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure), kube.RadixVolumeMountNameLabel, radixVolumeMountName) } -func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appName, namespace, componentName, pvcName, storageClassName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { +func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appName, namespace, componentName, pvcName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { requestsVolumeMountSize, err := resource.ParseQuantity(getRadixBlobFuse2VolumeMountRequestsStorage(radixVolumeMount)) if err != nil { requestsVolumeMountSize = resource.MustParse("1Mi") @@ -583,26 +549,28 @@ func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appNa Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceStorage: requestsVolumeMountSize}, // it seems correct number is not needed for CSI driver }, - VolumeName: pvcName, + VolumeName: pvName, }, } return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) } func populateCsiAzureVolumeMount(persistentVolume *corev1.PersistentVolume, appName string, volumeRootMount string, namespace string, componentName string, pvName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, provisioner string) error { - bindingMode := getBindingMode(getRadixBlobFuse2VolumeMountBindingMode(radixVolumeMount)) persistentVolume.ObjectMeta.Name = pvName persistentVolume.ObjectMeta.Labels = getCsiAzurePersistentVolumeLabels(appName, namespace, componentName, radixVolumeMount) persistentVolume.ObjectMeta.Annotations = getCsiAzurePersistentVolumeAnnotations(provisioner, secretName, namespace) - persistentVolume.Parameters = getCsiAzureStorageClassParameters(secretName, namespace, radixVolumeMount) - mountOptions, err := getCsiAzureStorageClassMountOptions(volumeRootMount, namespace, componentName, radixVolumeMount) + mountOptions, err := getCsiAzurePersistentVolumeMountOptions(volumeRootMount, namespace, componentName, radixVolumeMount) if err != nil { return err } persistentVolume.Spec.MountOptions = mountOptions + persistentVolume.Spec.CSI = &corev1.CSIPersistentVolumeSource{ + VolumeHandle: "azure-csi-driver-volume-handle", + Driver: "blob.csi.azure.com", + VolumeAttributes: getCsiAzurePersistentVolumeAttributes(radixVolumeMount), + } persistentVolume.Spec.CSI.NodePublishSecretRef = &corev1.SecretReference{Name: secretName, Namespace: namespace} // TODO - remove for Workload identity - persistentVolume.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimPolicy. PersistentVolumeReclaimPolicy deletes volume on unmount. - persistentVolume.VolumeBindingMode = &bindingMode + persistentVolume.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimRetain. PersistentVolumeReclaimPolicy deletes volume on unmount. return nil } @@ -614,13 +582,6 @@ func getCsiAzurePersistentVolumeAnnotations(provisioner, secretName, namespace s } } -func getBindingMode(bindingModeValue string) storagev1.VolumeBindingMode { - if strings.EqualFold(strings.ToLower(bindingModeValue), strings.ToLower(string(storagev1.VolumeBindingWaitForFirstConsumer))) { - return storagev1.VolumeBindingWaitForFirstConsumer - } - return storagev1.VolumeBindingImmediate -} - func getCsiAzurePersistentVolumeLabels(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { return map[string]string{ kube.RadixAppLabel: appName, @@ -631,45 +592,34 @@ func getCsiAzurePersistentVolumeLabels(appName, namespace, componentName string, } } -func getCsiAzureStorageClassParameters(secretName string, namespace string, radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { - parameters := map[string]string{ - csiStorageClassProvisionerSecretNameParameter: secretName, - csiStorageClassProvisionerSecretNamespaceParameter: namespace, - csiStorageClassNodeStageSecretNameParameter: secretName, - csiStorageClassNodeStageSecretNamespaceParameter: namespace, - } - skuName := getRadixBlobFuse2VolumeMountSkuName(radixVolumeMount) - if len(skuName) > 0 { - parameters[csiAzureStorageClassSkuNameParameter] = skuName - } +func getCsiAzurePersistentVolumeAttributes(radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { + attributes := make(map[string]string) switch GetCsiAzureVolumeMountType(radixVolumeMount) { case radixv1.MountTypeBlobFuse2FuseCsiAzure: - parameters[csiStorageClassContainerNameParameter] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) - parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterFuse + attributes[csiVolumeMountAttributeContainerName] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) + attributes[csiVolumeMountAttributeProtocol] = csiPersistentVolumeProtocolParameterFuse case radixv1.MountTypeBlobFuse2Fuse2CsiAzure: - parameters[csiStorageClassContainerNameParameter] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) - parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterFuse2 - case radixv1.MountTypeBlobFuse2NfsCsiAzure: - parameters[csiStorageClassContainerNameParameter] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) - parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterNfs - case radixv1.MountTypeAzureFileCsiAzure: - parameters[csiStorageClassShareNameParameter] = getRadixAzureFileVolumeMountShareName(radixVolumeMount) + attributes[csiVolumeMountAttributeContainerName] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) + attributes[csiVolumeMountAttributeProtocol] = csiPersistentVolumeProtocolParameterFuse2 + attributes[csiVolumeMountAttributeStorageAccount] = radixVolumeMount.BlobFuse2.StorageAccount + attributes[csiVolumeMountAttributeClientID] = radixVolumeMount.BlobFuse2.ClientId + attributes[csiVolumeMountAttributeResourceGroup] = radixVolumeMount.BlobFuse2.ResourceGroup } - return parameters + return attributes } -func getCsiAzureStorageClassMountOptions(volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) ([]string, error) { +func getCsiAzurePersistentVolumeMountOptions(volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) ([]string, error) { csiVolumeTypeId, err := getCsiRadixVolumeTypeIdForName(radixVolumeMount) if err != nil { return nil, err } - tmpPath := fmt.Sprintf(csiVolumeNodeMountPathTemplate, volumeRootMount, namespace, csiVolumeTypeId, componentName, radixVolumeMount.Name, GetRadixVolumeMountStorage(radixVolumeMount)) - return getCsiAzureStorageClassMountOptionsForAzureBlob(tmpPath, radixVolumeMount) + tmpPath := fmt.Sprintf(csiVolumeNodeMountPathTemplate, volumeRootMount, namespace, csiVolumeTypeId, componentName, radixVolumeMount.Name, getRadixVolumeMountStorage(radixVolumeMount)) + return getCsiAzurePersistentVolumeMountOptionsForAzureBlob(tmpPath, radixVolumeMount) } -func getCsiAzureStorageClassMountOptionsForAzureBlob(tmpPath string, radixVolumeMount *radixv1.RadixVolumeMount) ([]string, error) { +func getCsiAzurePersistentVolumeMountOptionsForAzureBlob(tmpPath string, radixVolumeMount *radixv1.RadixVolumeMount) ([]string, error) { mountOptions := []string{ - // fmt.Sprintf("--%s=%s", csiStorageClassTmpPathMountOption, tmpPath),//TODO fix this path to be able to mount on external mount + // fmt.Sprintf("--%s=%s", csiPersistentVolumeTmpPathMountOption, tmpPath),//TODO fix this path to be able to mount on external mount "--file-cache-timeout-in-seconds=120", "--use-attr-cache=true", "--cancel-list-on-mount-seconds=0", @@ -680,11 +630,11 @@ func getCsiAzureStorageClassMountOptionsForAzureBlob(tmpPath string, radixVolume } gid := getRadixBlobFuse2VolumeMountGid(radixVolumeMount) if len(gid) > 0 { - mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", csiStorageClassGidMountOption, gid)) + mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", csiPersistentVolumeGidMountOption, gid)) } else { uid := getRadixBlobFuse2VolumeMountUid(radixVolumeMount) if len(uid) > 0 { - mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", csiStorageClassUidMountOption, uid)) + mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", csiPersistentVolumeUidMountOption, uid)) } } if getRadixBlobFuse2VolumeMountAccessMode(radixVolumeMount) == string(corev1.ReadOnlyMany) { @@ -692,7 +642,7 @@ func getCsiAzureStorageClassMountOptionsForAzureBlob(tmpPath string, radixVolume } if radixVolumeMount.BlobFuse2 != nil { mountOptions = append(mountOptions, getStreamingMountOptions(radixVolumeMount.BlobFuse2.Streaming)...) - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassUseAdlsMountOption, radixVolumeMount.BlobFuse2.UseAdls != nil && *radixVolumeMount.BlobFuse2.UseAdls)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeUseAdlsMountOption, radixVolumeMount.BlobFuse2.UseAdls != nil && *radixVolumeMount.BlobFuse2.UseAdls)) } return mountOptions, nil } @@ -702,24 +652,24 @@ func getStreamingMountOptions(streaming *radixv1.RadixVolumeMountStreaming) []st if streaming != nil && streaming.Enabled != nil && !*streaming.Enabled { return nil } - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%t", csiStorageClassStreamingEnabledMountOption, true)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%t", csiPersistentVolumeStreamingEnabledMountOption, true)) if streaming == nil { return mountOptions } if streaming.StreamCache != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassStreamingCacheMountOption, *streaming.StreamCache)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeStreamingCacheMountOption, *streaming.StreamCache)) } if streaming.BlockSize != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassStreamingBlockSizeMountOption, *streaming.BlockSize)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeStreamingBlockSizeMountOption, *streaming.BlockSize)) } if streaming.BufferSize != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassStreamingBufferSizeMountOption, *streaming.BufferSize)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeStreamingBufferSizeMountOption, *streaming.BufferSize)) } if streaming.MaxBuffers != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassStreamingMaxBuffersMountOption, *streaming.MaxBuffers)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeStreamingMaxBuffersMountOption, *streaming.MaxBuffers)) } if streaming.MaxBlocksPerFile != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiStorageClassStreamingMaxBlocksPerFileMountOption, *streaming.MaxBlocksPerFile)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeStreamingMaxBlocksPerFileMountOption, *streaming.MaxBlocksPerFile)) } return mountOptions } @@ -759,13 +709,6 @@ func getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount *radixv1.RadixVo return radixVolumeMount.Storage } -func getRadixAzureFileVolumeMountShareName(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.AzureFile != nil { - return radixVolumeMount.AzureFile.Share - } - return radixVolumeMount.Storage -} - func getRadixBlobFuse2VolumeMountRequestsStorage(radixVolumeMount *radixv1.RadixVolumeMount) string { if radixVolumeMount.BlobFuse2 != nil { return radixVolumeMount.BlobFuse2.RequestsStorage @@ -788,11 +731,11 @@ func (deploy *Deployment) deletePersistentVolumeClaim(ctx context.Context, names return nil } -func (deploy *Deployment) deleteCsiAzureStorageClasses(ctx context.Context, storageClassName string) error { - if len(storageClassName) > 0 { - return deploy.kubeclient.StorageV1().StorageClasses().Delete(ctx, storageClassName, metav1.DeleteOptions{}) +func (deploy *Deployment) deleteCsiAzurePersistentVolumes(ctx context.Context, pvName string) error { + if len(pvName) > 0 { + return deploy.kubeclient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}) } - log.Ctx(ctx).Debug().Msg("Skip deleting StorageClass - name is empty") + log.Ctx(ctx).Debug().Msg("Skip deleting PersistentVolume - name is empty") return nil } @@ -804,8 +747,7 @@ func (deploy *Deployment) deletePersistentVolume(ctx context.Context, pvName str return nil } -// GetRadixVolumeMountStorage get RadixVolumeMount storage property, depend on volume type -func GetRadixVolumeMountStorage(radixVolumeMount *radixv1.RadixVolumeMount) string { +func getRadixVolumeMountStorage(radixVolumeMount *radixv1.RadixVolumeMount) string { if radixVolumeMount.Type == radixv1.MountTypeBlob { return radixVolumeMount.Container // Outdated } @@ -813,10 +755,6 @@ func GetRadixVolumeMountStorage(radixVolumeMount *radixv1.RadixVolumeMount) stri if len(blobFuse2VolumeMountContainer) != 0 { return blobFuse2VolumeMountContainer } - azureFileVolumeMountShare := getRadixAzureFileVolumeMountShareName(radixVolumeMount) - if len(azureFileVolumeMountShare) != 0 { - return azureFileVolumeMountShare - } return radixVolumeMount.Storage } @@ -851,13 +789,13 @@ func knownCSIDriver(csiPersistentVolumeSource *corev1.CSIPersistentVolumeSource) return ok } -// createOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - StorageClasses, PersistentVolumeClaims, PersistentVolume +// createOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims, PersistentVolume func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Context, desiredDeployment *appsv1.Deployment) error { namespace := deploy.radixDeployment.GetNamespace() appName := deploy.radixDeployment.Spec.AppName componentName := desiredDeployment.ObjectMeta.Name volumeRootMount := "/tmp" // TODO: add to environment variable, so this volume can be mounted to external disk - scList, err := deploy.getCsiAzureStorageClasses(ctx, namespace, componentName) + scList, err := deploy.getCsiAzurePersistentVolume(ctx, namespace, componentName) if err != nil { return err } @@ -869,7 +807,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont scMap := utils.GetPersistentVolumeMap(&scList.Items) pvcMap := utils.GetPersistentVolumeClaimMap(&pvcList.Items) radixVolumeMountMap := deploy.getRadixVolumeMountMapByCsiAzureVolumeMountName(componentName) - var actualStorageClassNames []string + var actualPersistentVolumeNames []string actualPvcNames, err := deploy.getCurrentlyUsedPersistentVolumeClaims(ctx, namespace) if err != nil { return err @@ -882,12 +820,12 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont if !existsRadixVolumeMount { return fmt.Errorf("not found Radix volume mount for desired volume %s", volume.Name) } - pv, storageClassIsCreated, err := deploy.getOrCreateCsiAzurePersistentVolume(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, volume.Name, scMap) + pv, pvIsCreated, err := deploy.getOrCreateCsiAzurePersistentVolume(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, volume.Name, scMap) if err != nil { return err } - actualStorageClassNames = append(actualStorageClassNames, pv.Name) - pvc, err := deploy.createCsiAzurePersistentVolumeClaim(ctx, pv, storageClassIsCreated, appName, namespace, componentName, radixVolumeMount, volume.PersistentVolumeClaim.ClaimName, pvcMap) + actualPersistentVolumeNames = append(actualPersistentVolumeNames, pv.Name) + pvc, err := deploy.createCsiAzurePersistentVolumeClaim(ctx, pv, pvIsCreated, appName, namespace, componentName, radixVolumeMount, volume.PersistentVolumeClaim.ClaimName, pvcMap) if err != nil { return err } @@ -967,42 +905,42 @@ func (deploy *Deployment) createCsiAzurePersistentVolumeClaim(ctx context.Contex if err != nil { return nil, err } - log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for StorageClass %s", persistentVolumeClaimName, namespace, pv.Name) + log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for PersistentVolume %s", persistentVolumeClaimName, namespace, pv.Name) return deploy.createPersistentVolumeClaim(ctx, appName, namespace, componentName, persistentVolumeClaimName, pv.Name, radixVolumeMount) } func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Context, appName, volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, volumeName string, pvMap map[string]*corev1.PersistentVolume) (*corev1.PersistentVolume, bool, error) { - var volumeMountProvisioner, foundProvisioner = getStorageClassProvisionerByVolumeMountType(radixVolumeMount) + var volumeMountProvisioner, foundProvisioner = getPersistentVolumeProvisionerByVolumeMountType(radixVolumeMount) if !foundProvisioner { return nil, false, fmt.Errorf("not found Storage Class provisioner for volume mount type %s", string(GetCsiAzureVolumeMountType(radixVolumeMount))) } - pvName := GetCsiAzureStorageClassName(namespace, volumeName) + pvName := GetCsiAzurePersistentVolumeName(namespace, volumeName) csiVolumeSecretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, radixVolumeMount.Name) - if existingStorageClass, exists := pvMap[pvName]; exists { - desiredStorageClass := existingStorageClass.DeepCopy() - err := populateCsiAzureVolumeMount(desiredStorageClass, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) + if existingPersistentVolume, exists := pvMap[pvName]; exists { + desiredPersistentVolume := existingPersistentVolume.DeepCopy() + err := populateCsiAzureVolumeMount(desiredPersistentVolume, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) if err != nil { return nil, false, err } - if equal, err := utils.EqualPersistentVolumes(existingStorageClass, desiredStorageClass); equal || err != nil { - return existingStorageClass, false, err + if equal, err := utils.EqualPersistentVolumes(existingPersistentVolume, desiredPersistentVolume); equal || err != nil { + return existingPersistentVolume, false, err } - log.Ctx(ctx).Info().Msgf("Delete StorageClass %s in namespace %s", existingStorageClass.Name, namespace) - err = deploy.deleteCsiAzureStorageClasses(ctx, existingStorageClass.Name) + log.Ctx(ctx).Info().Msgf("Delete PersistentVolume %s in namespace %s", existingPersistentVolume.Name, namespace) + err = deploy.deleteCsiAzurePersistentVolumes(ctx, existingPersistentVolume.Name) if err != nil { return nil, false, err } } - log.Ctx(ctx).Debug().Msgf("Create StorageClass %s in namespace %s", pvName, namespace) - storageClass := &storagev1.StorageClass{} - err := populateCsiAzureVolumeMount(storageClass, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) + log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) + pv := &corev1.PersistentVolume{} + err := populateCsiAzureVolumeMount(pv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) if err != nil { return nil, false, err } - desiredStorageClass, err := deploy.kubeclient.StorageV1().StorageClasses().Create(ctx, storageClass, metav1.CreateOptions{}) - return desiredStorageClass, true, err + desiredPersistentVolume, err := deploy.kubeclient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + return desiredPersistentVolume, true, err } func (deploy *Deployment) getRadixVolumeMountMapByCsiAzureVolumeMountName(componentName string) map[string]*radixv1.RadixVolumeMount { @@ -1025,7 +963,7 @@ func findCsiAzureVolumeForComponent(volumeMountMap map[string]*radixv1.RadixVolu return false } for _, radixVolumeMount := range volumeMounts { - if radixVolumeMount.BlobFuse2 == nil && radixVolumeMount.AzureFile == nil && !isKnownCsiAzureVolumeMount(string(GetCsiAzureVolumeMountType(&radixVolumeMount))) { + if radixVolumeMount.BlobFuse2 == nil && !isKnownCsiAzureVolumeMount(string(GetCsiAzureVolumeMountType(&radixVolumeMount))) { continue } radixVolumeMount := radixVolumeMount diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 78c1065a9..0e80c026a 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -22,7 +22,6 @@ import ( "github.com/stretchr/testify/suite" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -55,14 +54,14 @@ type volumeMountTestScenario struct { } type deploymentVolumesTestScenario struct { - name string - props expectedPvcScProperties - radixVolumeMounts []v1.RadixVolumeMount - volumes []corev1.Volume - existingStorageClassesBeforeTestRun []storagev1.StorageClass - existingPvcsBeforeTestRun []corev1.PersistentVolumeClaim - existingStorageClassesAfterTestRun []storagev1.StorageClass - existingPvcsAfterTestRun []corev1.PersistentVolumeClaim + name string + props expectedPvcScProperties + radixVolumeMounts []v1.RadixVolumeMount + volumes []corev1.Volume + existingPersistentVolumesBeforeTestRun []corev1.PersistentVolume + existingPvcsBeforeTestRun []corev1.PersistentVolumeClaim + existingPersistentVolumeAfterTestRun []corev1.PersistentVolume + existingPvcsAfterTestRun []corev1.PersistentVolumeClaim } type pvcTestScenario struct { @@ -119,69 +118,6 @@ func (suite *VolumeMountTestSuite) Test_NoVolumeMounts() { }) } -func (suite *VolumeMountTestSuite) Test_ValidFileCsiAzureVolumeMounts() { - scenarios := []volumeMountTestScenario{ - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume1", Storage: "storageName1", Path: "TestPath1"}, - expectedVolumeName: "csi-az-file-app-volume1-storageName1", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume2", Storage: "storageName2", Path: "TestPath2"}, - expectedVolumeName: "csi-az-file-app-volume2-storageName2", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "TestPath3"}, - expectedVolumeName: "csi-az-file-app-volume-with-long-name-storageName-with-lo-", - expectedVolumeNameIsPrefix: true, - }, - } - suite.T().Run("One File CSI Azure volume mount ", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - t.Logf("Test case %s for component %s", scenarios[0].name, factory.GetTargetType()) - component := utils.NewDeployCommonComponentBuilder(factory). - WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") - assert.Nil(t, err) - assert.Equal(t, 1, len(volumeMounts)) - if len(volumeMounts) > 0 { - mount := volumeMounts[0] - assert.Less(t, len(mount.Name), 64) - assert.Equal(t, scenarios[0].expectedVolumeName, mount.Name) - assert.Equal(t, scenarios[0].radixVolumeMount.Path, mount.MountPath) - } - } - }) - suite.T().Run("Multiple File CSI Azure volume mount", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - builder := utils.NewDeployCommonComponentBuilder(factory). - WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount, scenarios[1].radixVolumeMount, scenarios[2].radixVolumeMount) - - component := builder.BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") - assert.Nil(t, err) - assert.Equal(t, 3, len(volumeMounts)) - for idx, testCase := range scenarios { - if len(volumeMounts) > 0 { - assert.Less(t, len(volumeMounts[idx].Name), 64) - if testCase.expectedVolumeNameIsPrefix { - assert.True(t, strings.HasPrefix(volumeMounts[idx].Name, testCase.expectedVolumeName)) - } else { - assert.Equal(t, testCase.expectedVolumeName, volumeMounts[idx].Name) - } - assert.Equal(t, testCase.radixVolumeMount.Path, volumeMounts[idx].MountPath) - } - } - } - }) -} - func (suite *VolumeMountTestSuite) Test_ValidBlobCsiAzureVolumeMounts() { scenarios := []volumeMountTestScenario{ { @@ -340,12 +276,6 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { expectedVolumeName: "csi-az-blob-some-component-volume1-storage1", expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-volume1-storage1", }, - { - name: "File CSI Azure volume", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume1", Storage: "storage1", Path: "path1", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-volume1-storage1", - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-volume1-storage1", - }, { name: "Blob CSI Azure volume", radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "path1", GID: "1000"}, @@ -353,13 +283,6 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { expectedVolumeNameIsPrefix: true, expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-volume-with-long-name-storageN-", }, - { - name: "File CSI Azure volume", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "path1", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-volume-with-long-name-storageN-", - expectedVolumeNameIsPrefix: true, - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-volume-with-long-name-storageN-", - }, } blobFuseScenario := volumeMountTestScenario{ name: "Blob Azure FlexVolume", @@ -466,32 +389,6 @@ func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { pvc.Status.Phase = corev1.ClaimPending }), }, - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "File CSI Azure volume, PVS phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume3", Storage: "storage3", Path: "path3", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-volume3-storage3", - expectedPvcNamePrefix: "existing-file-pvc-name1", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeAzureFileCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Name = "existing-file-pvc-name1" - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "volume3" - pvc.Status.Phase = corev1.ClaimBound - }), - }, - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "File CSI Azure volume, PVS phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume4", Storage: "storage4", Path: "path4", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-volume4-storage4", - expectedPvcNamePrefix: "existing-file-pvc-name2", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeAzureFileCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Name = "existing-file-pvc-name2" - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "volume4" - pvc.Status.Phase = corev1.ClaimBound - }), - }, } suite.T().Run("CSI Azure volumes with existing PVC", func(t *testing.T) { @@ -553,24 +450,6 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { }, pvc: createPvc(namespace, componentName, v1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimPending }), }, - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "File CSI Azure volume, Status phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "file-volume1", Storage: "storage3", Path: "path3", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-file-volume1-storage3", - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-file-volume1-storage3", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeAzureFileCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimBound }), - }, - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "File CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "file-volume2", Storage: "storage4", Path: "path4", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-file-volume2-storage4", - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-file-volume2-storage4", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeAzureFileCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimPending }), - }, } suite.T().Run("No volumes", func(t *testing.T) { @@ -619,7 +498,7 @@ type expectedPvcScProperties struct { radixVolumeMountName string radixStorageName string pvcName string - storageClassName string + persistentVolumeName string radixVolumeMountType v1.MountType requestsVolumeMountSize string volumeAccessMode corev1.PersistentVolumeAccessMode @@ -649,18 +528,6 @@ func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { expectedVolumeName: "csi-az-blob-some-component-blob-volume2-storage2", expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume2-storage2", }, - { - name: "File CSI Azure volume, Status phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "file-volume1", Storage: "storage3", Path: "path3", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-file-volume1-storage3", - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-file-volume1-storage3", - }, - { - name: "File CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "file-volume2", Storage: "storage4", Path: "path4", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-file-volume2-storage4", - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-file-volume2-storage4", - }, { name: "Blob CSI Azure volume, Status phase: Pending", radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume-with-long-name", Storage: "storage-with-long-name", Path: "path2", GID: "1000"}, @@ -668,13 +535,6 @@ func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { expectedVolumeNameIsPrefix: true, expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume-with-long-name-", }, - { - name: "File CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "file-volume-with-long-name", Storage: "storage-with-long-name", Path: "path4", GID: "1000"}, - expectedVolumeName: "csi-az-file-some-component-file-volume-with-long-name-sto-", - expectedVolumeNameIsPrefix: true, - expectedPvcNamePrefix: "pvc-csi-az-file-some-component-file-volume-with-long-name-", - }, } suite.T().Run("No volumes", func(t *testing.T) { @@ -720,67 +580,6 @@ func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { }) } -func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureBlobVolumeResources() { - scenarios := []volumeMountTestScenario{ - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume1", Storage: "storageName1", Path: "TestPath1"}, - expectedVolumeName: "csi-az-file-app-volume1-storageName1", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume2", Storage: "storageName2", Path: "TestPath2"}, - expectedVolumeName: "csi-az-file-app-volume2-storageName2", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeAzureFileCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "TestPath3"}, - expectedVolumeName: "csi-az-file-app-volume-with-long-name-storageName-with-lo-", - expectedVolumeNameIsPrefix: true, - }, - } - suite.T().Run("One File CSI Azure volume mount ", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - t.Logf("Test case %s for component %s", scenarios[0].name, factory.GetTargetType()) - component := utils.NewDeployCommonComponentBuilder(factory). - WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") - assert.Nil(t, err) - assert.Equal(t, 1, len(volumeMounts)) - if len(volumeMounts) > 0 { - mount := volumeMounts[0] - assert.Equal(t, scenarios[0].expectedVolumeName, mount.Name) - assert.Equal(t, scenarios[0].radixVolumeMount.Path, mount.MountPath) - } - } - }) - suite.T().Run("Multiple File CSI Azure volume mount", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { - component := utils.NewDeployCommonComponentBuilder(factory). - WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount, scenarios[1].radixVolumeMount, scenarios[2].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(component, "") - assert.Nil(t, err) - assert.Equal(t, 3, len(volumeMounts)) - for idx, testCase := range scenarios { - if len(volumeMounts) > 0 { - assert.Less(t, len(volumeMounts[idx].Name), 64) - if testCase.expectedVolumeNameIsPrefix { - assert.True(t, strings.HasPrefix(volumeMounts[idx].Name, testCase.expectedVolumeName)) - } else { - assert.Equal(t, testCase.expectedVolumeName, volumeMounts[idx].Name) - } - assert.Equal(t, testCase.radixVolumeMount.Path, volumeMounts[idx].MountPath) - } - } - } - }) -} - func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { appName := "any-app" environment := "some-env" @@ -802,15 +601,14 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{}, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) {}), + existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, + existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), }, } } return []deploymentVolumesTestScenario{ getScenario(getPropsCsiBlobVolume1Storage1(nil)), - getScenario(getPropsCsiFileVolume2Storage2(nil)), } }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { @@ -820,7 +618,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { expectedVolumeName string expectedNewSecretName string expectedNewPvcName string - expectedNewStorageClassName string + expectedNewPvName string expectedNewScTmpPath string } getScenario := func(props expectedPvcScProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { @@ -845,20 +643,20 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pvc.Spec.StorageClassName = utils.StringPtr(scenarioProps.expectedNewStorageClassName) + pvc.Spec.VolumeName = scenarioProps.expectedNewPvName }), }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) {}), + existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), }, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { - sc.ObjectMeta.Name = scenarioProps.expectedNewStorageClassName + existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) { + sc.ObjectMeta.Name = scenarioProps.expectedNewPvName sc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // setStorageClassMountOption(sc, "--tmp-path", scenarioProps.expectedNewScTmpPath) //TODO: this option does not work with blobfuse2 in some reason - investigate to make use separate disk volume for csi volumes - setStorageClassStorageParameter(props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, sc) - sc.Parameters[csiStorageClassProvisionerSecretNameParameter] = scenarioProps.expectedNewSecretName - sc.Parameters[csiStorageClassNodeStageSecretNameParameter] = scenarioProps.expectedNewSecretName + // setPersistentVolumeMountOption(sc, "--tmp-path", scenarioProps.expectedNewScTmpPath) //TODO: this option does not work with blobfuse2 in some reason - investigate to make use separate disk volume for csi volumes + setVolumeMountAttribute(props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, sc) + sc.Spec.CSI.VolumeAttributes[csiPersistentVolumeProvisionerSecretNameParameter] = scenarioProps.expectedNewSecretName + sc.Spec.CSI.VolumeAttributes[csiPersistentVolumeNodeStageSecretNameParameter] = scenarioProps.expectedNewSecretName }), }, } @@ -870,28 +668,19 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", expectedNewSecretName: "some-component-volume101-csiazurecreds", expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - expectedNewStorageClassName: "sc-any-app-some-env-csi-az-blob-some-component-volume101-storage101", + expectedNewPvName: "sc-any-app-some-env-csi-az-blob-some-component-volume101-storage101", expectedNewScTmpPath: "/tmp/any-app-some-env/csi-az-blob/some-component/volume101/storage101", }), - getScenario(getPropsCsiFileVolume2Storage2(nil), scenarioProperties{ - changedNewRadixVolumeName: "volume101", - changedNewRadixVolumeStorageName: "storage101", - expectedVolumeName: "csi-az-file-some-component-volume101-storage101", - expectedNewSecretName: "some-component-volume101-csiazurecreds", - expectedNewPvcName: "pvc-csi-az-file-some-component-volume101-storage101-12345", - expectedNewStorageClassName: "sc-any-app-some-env-csi-az-file-some-component-volume101-storage101", - expectedNewScTmpPath: "/tmp/any-app-some-env/csi-az-file/some-component/volume101/storage101", - }), } }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - storageClassForAnotherNamespace := createRandomStorageClass(props, utils.RandString(10), utils.RandString(10)) - storageClassForAnotherComponent := createRandomStorageClass(props, props.namespace, utils.RandString(10)) + persistentVolumeForAnotherNamespace := createRandomPersistentVolume(props, utils.RandString(10), utils.RandString(10)) + persistentVolumeForAnotherComponent := createRandomPersistentVolume(props, props.namespace, utils.RandString(10)) pvcForAnotherNamespace := createRandomPvc(props, utils.RandString(10), utils.RandString(10)) pvcForAnotherComponent := createRandomPvc(props, props.namespace, utils.RandString(10)) return deploymentVolumesTestScenario{ - name: "Garbage collect orphaned PVCs and StorageClasses", + name: "Garbage collect orphaned PVCs and PersistentVolume", props: props, radixVolumeMounts: []v1.RadixVolumeMount{ createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), @@ -909,21 +698,20 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { pvcForAnotherNamespace, pvcForAnotherComponent, }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{ - createRandomStorageClass(props, props.namespace, props.componentName), - storageClassForAnotherNamespace, - storageClassForAnotherComponent, + existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + createRandomPersistentVolume(props, props.namespace, props.componentName), + persistentVolumeForAnotherNamespace, + persistentVolumeForAnotherComponent, }, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) {}), - storageClassForAnotherNamespace, - storageClassForAnotherComponent, + existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), + persistentVolumeForAnotherNamespace, + persistentVolumeForAnotherComponent, }, } } return []deploymentVolumesTestScenario{ getScenario(getPropsCsiBlobVolume1Storage1(nil)), - getScenario(getPropsCsiFileVolume2Storage2(nil)), } }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { @@ -945,19 +733,18 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} }), }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{ - createRandomStorageClass(props, props.namespace, props.componentName), + existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + createRandomPersistentVolume(props, props.namespace, props.componentName), }, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { - sc.MountOptions = append(sc.MountOptions, "-o ro") + existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") }), }, } } return []deploymentVolumesTestScenario{ getScenario(getPropsCsiBlobVolume1Storage1(nil)), - getScenario(getPropsCsiFileVolume2Storage2(nil)), } }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { @@ -979,17 +766,16 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} }), }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{ - createRandomStorageClass(props, props.namespace, props.componentName), + existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + createRandomPersistentVolume(props, props.namespace, props.componentName), }, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) {}), + existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), }, } } return []deploymentVolumesTestScenario{ getScenario(getPropsCsiBlobVolume1Storage1(nil)), - getScenario(getPropsCsiFileVolume2Storage2(nil)), } }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { @@ -1011,17 +797,16 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} }), }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{ - createRandomStorageClass(props, props.namespace, props.componentName), + existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + createRandomPersistentVolume(props, props.namespace, props.componentName), }, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) {}), + existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), }, } } return []deploymentVolumesTestScenario{ getScenario(getPropsCsiBlobVolume1Storage1(nil)), - getScenario(getPropsCsiFileVolume2Storage2(nil)), } }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { @@ -1039,10 +824,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{}, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { - sc.MountOptions = []string{ + existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, + existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = []string{ "--file-cache-timeout-in-seconds=120", "--use-attr-cache=true", "--cancel-list-on-mount-seconds=0", @@ -1086,10 +871,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{}, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { - sc.MountOptions = []string{ + existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, + existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = []string{ "--file-cache-timeout-in-seconds=120", "--use-attr-cache=true", "--cancel-list-on-mount-seconds=0", @@ -1139,10 +924,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), }, - existingStorageClassesBeforeTestRun: []storagev1.StorageClass{}, - existingStorageClassesAfterTestRun: []storagev1.StorageClass{ - createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { - sc.MountOptions = []string{ + existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, + existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = []string{ "--file-cache-timeout-in-seconds=120", "--use-attr-cache=true", "--cancel-list-on-mount-seconds=0", @@ -1162,7 +947,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { } }()...) - suite.T().Run("CSI Azure volume PVCs and StorageClasses", func(t *testing.T) { + suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { t.Parallel() for _, factory := range suite.radixCommonDeployComponentFactories { for _, scenario := range scenarios { @@ -1177,14 +962,14 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), desiredDeployment) assert.Nil(t, err) - existingPvcs, existingScs, err := getExistingPvcsAndStorageClassesFromFakeCluster(deployment) + existingPvcs, existingScs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment) assert.Nil(t, err) equalPvcLists, err := utils.EqualPvcLists(&scenario.existingPvcsAfterTestRun, &existingPvcs, true) assert.Nil(t, err) assert.True(t, equalPvcLists) - equalStorageClassLists, err := utils.EqualPersistentVolumeLists(&scenario.existingStorageClassesAfterTestRun, &existingScs) + equalPersistentVolumeLists, err := utils.EqualPersistentVolumeLists(&scenario.existingPersistentVolumeAfterTestRun, &existingScs) assert.Nil(t, err) - assert.True(t, equalStorageClassLists) + assert.True(t, equalPersistentVolumeLists) } } }) @@ -1404,8 +1189,8 @@ func Test_EmptyDir(t *testing.T) { } -func createRandomStorageClass(props expectedPvcScProperties, namespace, componentName string) storagev1.StorageClass { - return createExpectedStorageClass(props, func(sc *storagev1.StorageClass) { +func createRandomPersistentVolume(props expectedPvcScProperties, namespace, componentName string) corev1.PersistentVolume { + return createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) { sc.ObjectMeta.Name = utils.RandString(10) sc.ObjectMeta.Labels[kube.RadixNamespace] = namespace sc.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName @@ -1417,12 +1202,12 @@ func createRandomPvc(props expectedPvcScProperties, namespace, componentName str pvc.ObjectMeta.Name = utils.RandString(10) pvc.ObjectMeta.Namespace = namespace pvc.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName - pvc.Spec.StorageClassName = utils.StringPtr(utils.RandString(10)) + pvc.Spec.VolumeName = utils.RandString(10) }) } // TODO: this option does not work with blobfuse2 in some reason - investigate to make use separate disk volume for csi volumes -// func setStorageClassMountOption(sc *storagev1.StorageClass, key, value string) { +// func setPersistentVolumeMountOption(sc *corev1.PersistentVolume, key, value string) { // mountOptions := sc.MountOptions // for i, option := range mountOptions { // if strings.Contains(option, key) { @@ -1445,7 +1230,7 @@ func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcScProperties)) expec radixVolumeMountName: "volume1", radixStorageName: "storage1", pvcName: "pvc-csi-az-blob-some-component-volume1-storage1-12345", - storageClassName: "sc-any-app-some-env-csi-az-blob-some-component-volume1-storage1", + persistentVolumeName: "sc-any-app-some-env-csi-az-blob-some-component-volume1-storage1", radixVolumeMountType: v1.MountTypeBlobFuse2FuseCsiAzure, requestsVolumeMountSize: "1Mi", volumeAccessMode: corev1.ReadOnlyMany, // default access mode @@ -1474,7 +1259,7 @@ func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcScProperties)) radixVolumeMountName: "volume1", radixStorageName: "storage1", pvcName: "pvc-csi-blobfuse2-fuse2-some-component-volume1-storage1-12345", - storageClassName: "sc-any-app-some-env-csi-blobfuse2-fuse2-some-component-volume1-storage1", + persistentVolumeName: "sc-any-app-some-env-csi-blobfuse2-fuse2-some-component-volume1-storage1", radixVolumeMountType: v1.MountTypeBlobFuse2Fuse2CsiAzure, requestsVolumeMountSize: "1Mi", volumeAccessMode: corev1.ReadOnlyMany, // default access mode @@ -1491,62 +1276,33 @@ func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcScProperties)) return props } -func getPropsCsiFileVolume2Storage2(modify func(*expectedPvcScProperties)) expectedPvcScProperties { - appName := "any-app" - environment := "some-env" - componentName := "some-component" - props := expectedPvcScProperties{ - appName: appName, - environment: environment, - namespace: fmt.Sprintf("%s-%s", appName, environment), - componentName: componentName, - radixVolumeMountName: "volume2", - radixStorageName: "storage2", - pvcName: "pvc-csi-az-file-some-component-volume2-storage2-12345", - storageClassName: "sc-any-app-some-env-csi-az-file-some-component-volume2-storage2", - radixVolumeMountType: v1.MountTypeAzureFileCsiAzure, - requestsVolumeMountSize: "1Mi", - volumeAccessMode: corev1.ReadOnlyMany, // default access mode - volumeName: "csi-az-file-some-component-volume2-storage2", - scProvisioner: provisionerFileCsiAzure, - scSecretName: "some-component-volume2-csiazurecreds", - scTmpPath: "/tmp/any-app-some-env/csi-az-file/some-component/volume2/storage2", - scGid: "1000", - scUid: "", - } - if modify != nil { - modify(&props) - } - return props -} - func putExistingDeploymentVolumesScenarioDataToFakeCluster(scenario *deploymentVolumesTestScenario, deployment *Deployment) { for _, pvc := range scenario.existingPvcsBeforeTestRun { _, _ = deployment.kubeclient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.Background(), &pvc, metav1.CreateOptions{}) } - for _, sc := range scenario.existingStorageClassesBeforeTestRun { - _, _ = deployment.kubeclient.StorageV1().StorageClasses().Create(context.Background(), &sc, metav1.CreateOptions{}) + for _, pv := range scenario.existingPersistentVolumesBeforeTestRun { + _, _ = deployment.kubeclient.CoreV1().PersistentVolumes().Create(context.Background(), &pv, metav1.CreateOptions{}) } } -func getExistingPvcsAndStorageClassesFromFakeCluster(deployment *Deployment) ([]corev1.PersistentVolumeClaim, []storagev1.StorageClass, error) { +func getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment *Deployment) ([]corev1.PersistentVolumeClaim, []corev1.PersistentVolume, error) { var pvcItems []corev1.PersistentVolumeClaim - var scItems []storagev1.StorageClass + var pvItems []corev1.PersistentVolume pvcList, err := deployment.kubeclient.CoreV1().PersistentVolumeClaims("").List(context.Background(), metav1.ListOptions{}) if err != nil { - return pvcItems, scItems, err + return pvcItems, pvItems, err } if pvcList != nil && pvcList.Items != nil { pvcItems = pvcList.Items } - storageClassList, err := deployment.kubeclient.StorageV1().StorageClasses().List(context.Background(), metav1.ListOptions{}) + pvList, err := deployment.kubeclient.CoreV1().PersistentVolumes().List(context.Background(), metav1.ListOptions{}) if err != nil { - return pvcItems, scItems, err + return pvcItems, pvItems, err } - if storageClassList != nil && storageClassList.Items != nil { - scItems = storageClassList.Items + if pvList != nil && pvList.Items != nil { + pvItems = pvList.Items } - return pvcItems, scItems, nil + return pvcItems, pvItems, nil } func getDesiredDeployment(componentName string, volumes []corev1.Volume) *appsv1.Deployment { @@ -1610,7 +1366,7 @@ func buildRdWithComponentBuilders(appName string, environment string, componentB BuildRD() } -func createExpectedStorageClass(props expectedPvcScProperties, modify func(class *storagev1.StorageClass)) storagev1.StorageClass { +func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *corev1.PersistentVolume)) corev1.PersistentVolume { mountOptions := []string{ "--file-cache-timeout-in-seconds=120", "--use-attr-cache=true", @@ -1621,15 +1377,14 @@ func createExpectedStorageClass(props expectedPvcScProperties, modify func(class "-o negative_timeout=120", // fmt.Sprintf("--tmp-path=%s", props.scTmpPath), //TODO: this option does not work with blobfuse2 in some reason - investigate } - idOption := getStorageClassIdMountOption(props) + idOption := getPersistentVolumeIdMountOption(props) if len(idOption) > 0 { mountOptions = append(mountOptions, idOption) } reclaimPolicy := corev1.PersistentVolumeReclaimRetain - bindingMode := storagev1.VolumeBindingImmediate - sc := storagev1.StorageClass{ + sc := corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ - Name: props.storageClassName, + Name: props.persistentVolumeName, Labels: map[string]string{ kube.RadixAppLabel: props.appName, kube.RadixNamespace: props.namespace, @@ -1638,41 +1393,37 @@ func createExpectedStorageClass(props expectedPvcScProperties, modify func(class kube.RadixVolumeMountNameLabel: props.radixVolumeMountName, }, }, - Provisioner: props.scProvisioner, - Parameters: map[string]string{ - csiStorageClassProvisionerSecretNameParameter: props.scSecretName, - csiStorageClassProvisionerSecretNamespaceParameter: props.namespace, - csiStorageClassNodeStageSecretNameParameter: props.scSecretName, - csiStorageClassNodeStageSecretNamespaceParameter: props.namespace, + Spec: corev1.PersistentVolumeSpec{ + Provisioner: props.scProvisioner, + Attr: map[string]string{ + csiPersistentVolumeProvisionerSecretNameParameter: props.scSecretName, + csiPersistentVolumeProvisionerSecretNamespaceParameter: props.namespace, + csiPersistentVolumeNodeStageSecretNameParameter: props.scSecretName, + csiPersistentVolumeNodeStageSecretNamespaceParameter: props.namespace, + }, + MountOptions: mountOptions, + ReclaimPolicy: &reclaimPolicy, }, - MountOptions: mountOptions, - ReclaimPolicy: &reclaimPolicy, - VolumeBindingMode: &bindingMode, } - setStorageClassStorageParameter(props.radixVolumeMountType, props.radixStorageName, &sc) + setVolumeMountAttribute(props.radixVolumeMountType, props.radixStorageName, &sc) if modify != nil { modify(&sc) } return sc } -func setStorageClassStorageParameter(radixVolumeMountType v1.MountType, storageName string, sc *storagev1.StorageClass) { +func setVolumeMountAttribute(radixVolumeMountType v1.MountType, containerName string, pv *corev1.PersistentVolume) { switch radixVolumeMountType { case v1.MountTypeBlobFuse2FuseCsiAzure: - sc.Parameters[csiStorageClassContainerNameParameter] = storageName - sc.Parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterFuse + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeContainerName] = containerName + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeProtocol] = csiPersistentVolumeProtocolParameterFuse case v1.MountTypeBlobFuse2Fuse2CsiAzure: - sc.Parameters[csiStorageClassContainerNameParameter] = storageName - sc.Parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterFuse2 - case v1.MountTypeBlobFuse2NfsCsiAzure: - sc.Parameters[csiStorageClassContainerNameParameter] = storageName - sc.Parameters[csiStorageClassProtocolParameter] = csiStorageClassProtocolParameterNfs - case v1.MountTypeAzureFileCsiAzure: - sc.Parameters[csiStorageClassShareNameParameter] = storageName + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeContainerName] = containerName + pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeProtocol] = csiPersistentVolumeProtocolParameterFuse2 } } -func getStorageClassIdMountOption(props expectedPvcScProperties) string { +func getPersistentVolumeIdMountOption(props expectedPvcScProperties) string { if len(props.scGid) > 0 { return fmt.Sprintf("-o gid=%s", props.scGid) } @@ -1700,7 +1451,7 @@ func createExpectedPvc(props expectedPvcScProperties, modify func(*corev1.Persis Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, // it seems correct number is not needed for CSI driver }, - StorageClassName: utils.StringPtr(props.storageClassName), + PersistentVolumeName: utils.StringPtr(props.persistentVolumeName), }, } if modify != nil { diff --git a/pkg/apis/metrics/custom_metrics.go b/pkg/apis/metrics/custom_metrics.go index 355dd0e0b..6869b7659 100644 --- a/pkg/apis/metrics/custom_metrics.go +++ b/pkg/apis/metrics/custom_metrics.go @@ -5,14 +5,13 @@ import ( "fmt" "time" - "github.com/equinor/radix-operator/pkg/apis/utils" - "k8s.io/apimachinery/pkg/api/resource" - "github.com/equinor/radix-operator/pkg/apis/defaults" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" ) var ( diff --git a/pkg/apis/radix/v1/radixapptypes.go b/pkg/apis/radix/v1/radixapptypes.go index 9dc5f4604..c8bfef50d 100644 --- a/pkg/apis/radix/v1/radixapptypes.go +++ b/pkg/apis/radix/v1/radixapptypes.go @@ -824,8 +824,8 @@ type RadixPrivateImageHubCredential struct { // RadixVolumeMount defines an external storage resource. type RadixVolumeMount struct { // Type defines the storage type. - // Deprecated, use BlobFuse2 or AzureFile instead. - // +kubebuilder:validation:Enum=blob;azure-blob;azure-file;"" + // Deprecated, use BlobFuse2 instead. + // +kubebuilder:validation:Enum=blob;azure-blob;"" // +optional Type MountType `json:"type"` @@ -840,7 +840,7 @@ type RadixVolumeMount struct { Container string `json:"container,omitempty"` // Outdated. Use Storage instead // Storage defines the name of the container in the external storage resource. - // Deprecated, use BlobFuse2 or AzureFile instead. + // Deprecated, use BlobFuse2 instead. // +optional Storage string `json:"storage"` // Container name, file Share name, etc. @@ -849,37 +849,37 @@ type RadixVolumeMount struct { Path string `json:"path"` // Path within the pod (replica), where the volume mount has been mounted to // GID defines the group ID (number) which will be set as owner of the mounted volume. - // Deprecated, use BlobFuse2 or AzureFile instead. + // Deprecated, use BlobFuse2 instead. // +optional GID string `json:"gid,omitempty"` // Optional. Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting. https://github.com/kubernetes-sigs/blob-csi-driver/blob/master/docs/driver-parameters.md // UID defines the user ID (number) which will be set as owner of the mounted volume. - // Deprecated, use BlobFuse2 or AzureFile instead. + // Deprecated, use BlobFuse2 instead. // +optional UID string `json:"uid,omitempty"` // Optional. Volume mount owner UserID. Used instead of GID. // TODO: describe // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 or AzureFile instead. + // Deprecated, use BlobFuse2 instead. // +optional SkuName string `json:"skuName,omitempty"` // Available values: Standard_LRS (default), Premium_LRS, Standard_GRS, Standard_RAGRS. https://docs.microsoft.com/en-us/rest/api/storagerp/srp_sku_types // TODO: describe // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 or AzureFile instead. + // Deprecated, use BlobFuse2 instead. // +optional RequestsStorage string `json:"requestsStorage,omitempty"` // Requests resource storage size. Default "1Mi". https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim // Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 or AzureFile instead. + // Deprecated, use BlobFuse2 instead. // +kubebuilder:validation:Enum=ReadOnlyMany;ReadWriteOnce;ReadWriteMany;"" // +optional AccessMode string `json:"accessMode,omitempty"` // Available values: ReadOnlyMany (default) - read-only by many nodes, ReadWriteOnce - read-write by a single node, ReadWriteMany - read-write by many nodes. https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes // Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 or AzureFile instead. + // Deprecated, use BlobFuse2 instead. // +kubebuilder:validation:Enum=Immediate;WaitForFirstConsumer;"" // +optional BindingMode string `json:"bindingMode,omitempty"` // Volume binding mode. Available values: Immediate (default), WaitForFirstConsumer. https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode @@ -888,6 +888,7 @@ type RadixVolumeMount struct { BlobFuse2 *RadixBlobFuse2VolumeMount `json:"blobFuse2,omitempty"` // AzureFile settings for Azure File CSI driver + // Deprecated, use BlobFuse2 instead. AzureFile *RadixAzureFileVolumeMount `json:"azureFile,omitempty"` // EmptyDir settings for EmptyDir volume @@ -902,10 +903,6 @@ func (v *RadixVolumeMount) HasBlobFuse2() bool { return v.BlobFuse2 != nil } -func (v *RadixVolumeMount) HasAzureFile() bool { - return v.AzureFile != nil -} - func (v *RadixVolumeMount) HasEmptyDir() bool { return v.EmptyDir != nil } @@ -923,15 +920,13 @@ type BlobFuse2Protocol string const ( // BlobFuse2ProtocolFuse2 Use of fuse2 protocol for storage account for blobfuse2 BlobFuse2ProtocolFuse2 BlobFuse2Protocol = "fuse2" - // BlobFuse2ProtocolNfs Use of NFS storage account for blobfuse2 - BlobFuse2ProtocolNfs BlobFuse2Protocol = "nfs" ) // RadixBlobFuse2VolumeMount defines an external storage resource, configured to use Blobfuse2 - A Microsoft supported Azure Storage FUSE driver. // More info: https://github.com/Azure/azure-storage-fuse type RadixBlobFuse2VolumeMount struct { // Holds protocols of BlobFuse2 Azure Storage FUSE driver. Default is fuse2. - // +kubebuilder:validation:Enum=fuse2;nfs;"" + // +kubebuilder:validation:Enum=fuse2;"" // +optional Protocol BlobFuse2Protocol `json:"protocol,omitempty"` @@ -978,11 +973,22 @@ type RadixBlobFuse2VolumeMount struct { // More info: https://github.com/Azure/azure-storage-fuse/blob/main/STREAMING.md // +optional Streaming *RadixVolumeMountStreaming `json:"streaming,omitempty"` // Optional. Streaming configuration. Used for blobfuse2. + + // Name of a storage account. If not defined - it will be configured in a secret. + // +optional + StorageAccount string + + // ClientID of a service principal. Applicable when using a workload identity. + // +optional + ClientId string + + // ResourceGroup of a storage account. Applicable when using a workload identity. + // +optional + ResourceGroup string } // RadixAzureFileVolumeMount defines an external storage resource, configured to use Azure File with CSI driver. -// More info: https://github.com/kubernetes-sigs/azurefile-csi-driver -// https://github.com/kubernetes-sigs/azurefile-csi-driver/blob/master/docs/driver-parameters.md +// Deprecated, use BlobFuse2 instead. type RadixAzureFileVolumeMount struct { // Share. Name of the file share in the external storage resource. // +optional @@ -1058,10 +1064,6 @@ const ( MountTypeBlobFuse2FuseCsiAzure MountType = "azure-blob" // MountTypeBlobFuse2Fuse2CsiAzure Use of azure/csi driver for blobfuse2, protocol Fuse2 in Azure storage account MountTypeBlobFuse2Fuse2CsiAzure MountType = "blobfuse2-fuse2" - // MountTypeBlobFuse2NfsCsiAzure Use of azure/csi driver for blobfuse2, protocol NFS in Azure storage account - MountTypeBlobFuse2NfsCsiAzure MountType = "blobfuse2-nfs" - // MountTypeAzureFileCsiAzure Use of azure/csi driver for Azure File in Azure storage account - MountTypeAzureFileCsiAzure MountType = "azure-file" ) // RadixNode defines node attributes, where container should be scheduled diff --git a/pkg/apis/radixvalidators/errors.go b/pkg/apis/radixvalidators/errors.go index 43ffdedc5..275c00313 100644 --- a/pkg/apis/radixvalidators/errors.go +++ b/pkg/apis/radixvalidators/errors.go @@ -287,10 +287,6 @@ func volumeMountBlobFuse2ValidationError(cause error) error { return fmt.Errorf("blobFuse2 failed validation. %w", cause) } -func volumeMountAzureFileValidationError(cause error) error { - return fmt.Errorf("azureFile failed validation. %w", cause) -} - func volumeMountEmptyDirValidationError(cause error) error { return fmt.Errorf("emptyDir failed validation. %w", cause) } diff --git a/pkg/apis/radixvalidators/validate_ra.go b/pkg/apis/radixvalidators/validate_ra.go index 8f1e49c84..0367e075e 100644 --- a/pkg/apis/radixvalidators/validate_ra.go +++ b/pkg/apis/radixvalidators/validate_ra.go @@ -1474,7 +1474,7 @@ func validateVolumeMounts(volumeMounts []radixv1.RadixVolumeMount) error { } volumeSourceCount := len(slice.FindAll( - []bool{v.HasDeprecatedVolume(), v.HasBlobFuse2(), v.HasAzureFile(), v.HasEmptyDir()}, + []bool{v.HasDeprecatedVolume(), v.HasBlobFuse2(), v.HasEmptyDir()}, func(b bool) bool { return b }), ) if volumeSourceCount > 1 { @@ -1493,10 +1493,6 @@ func validateVolumeMounts(volumeMounts []radixv1.RadixVolumeMount) error { if err := validateVolumeMountBlobFuse2(v.BlobFuse2); err != nil { return volumeMountValidationError(v.Name, err) } - case v.HasAzureFile(): - if err := validateVolumeMountAzureFile(v.AzureFile); err != nil { - return volumeMountValidationError(v.Name, err) - } case v.HasEmptyDir(): if err := validateVolumeMountEmptyDir(v.EmptyDir); err != nil { return volumeMountValidationError(v.Name, err) @@ -1508,7 +1504,7 @@ func validateVolumeMounts(volumeMounts []radixv1.RadixVolumeMount) error { } func validateVolumeMountDeprecatedSource(v *radixv1.RadixVolumeMount) error { - if !slices.Contains([]radixv1.MountType{radixv1.MountTypeBlob, radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeAzureFileCsiAzure}, v.Type) { + if !slices.Contains([]radixv1.MountType{radixv1.MountTypeBlob, radixv1.MountTypeBlobFuse2FuseCsiAzure}, v.Type) { return volumeMountDeprecatedSourceValidationError(ErrVolumeMountInvalidType) } @@ -1523,7 +1519,7 @@ func validateVolumeMountDeprecatedSource(v *radixv1.RadixVolumeMount) error { if len(v.Container) == 0 { return volumeMountDeprecatedSourceValidationError(ErrVolumeMountMissingContainer) } - case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeAzureFileCsiAzure: + case radixv1.MountTypeBlobFuse2FuseCsiAzure: if len(v.Storage) == 0 { return volumeMountDeprecatedSourceValidationError(ErrVolumeMountMissingStorage) } @@ -1533,7 +1529,7 @@ func validateVolumeMountDeprecatedSource(v *radixv1.RadixVolumeMount) error { } func validateVolumeMountBlobFuse2(fuse2 *radixv1.RadixBlobFuse2VolumeMount) error { - if !slices.Contains([]radixv1.BlobFuse2Protocol{radixv1.BlobFuse2ProtocolFuse2, radixv1.BlobFuse2ProtocolNfs, ""}, fuse2.Protocol) { + if !slices.Contains([]radixv1.BlobFuse2Protocol{radixv1.BlobFuse2ProtocolFuse2, ""}, fuse2.Protocol) { return volumeMountBlobFuse2ValidationError(ErrVolumeMountInvalidProtocol) } @@ -1549,10 +1545,6 @@ func validateVolumeMountBlobFuse2(fuse2 *radixv1.RadixBlobFuse2VolumeMount) erro return nil } -func validateVolumeMountAzureFile(_ *radixv1.RadixAzureFileVolumeMount) error { - return volumeMountAzureFileValidationError(ErrVolumeMountTypeNotImplemented) -} - func validateVolumeMountEmptyDir(emptyDir *radixv1.RadixEmptyDirVolumeMount) error { if emptyDir.SizeLimit.IsZero() { return volumeMountEmptyDirValidationError(ErrVolumeMountMissingSizeLimit) diff --git a/pkg/apis/radixvalidators/validate_ra_test.go b/pkg/apis/radixvalidators/validate_ra_test.go index b96a03899..8a1bd3876 100644 --- a/pkg/apis/radixvalidators/validate_ra_test.go +++ b/pkg/apis/radixvalidators/validate_ra_test.go @@ -1459,24 +1459,6 @@ func Test_ValidationOfVolumeMounts_Errors(t *testing.T) { updateRA: setComponentAndJobsVolumeMounts, expectedError: nil, }, - "blobfuse2: valid protocol nfs": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Name: "some_name", - Path: "some_path", - BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{ - Protocol: radixv1.BlobFuse2ProtocolNfs, - Container: "any-container", - }, - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: nil, - }, "blobfuse2: valid requestsStorage": { volumeMounts: func() []radixv1.RadixVolumeMount { volumeMounts := []radixv1.RadixVolumeMount{ @@ -1546,21 +1528,6 @@ func Test_ValidationOfVolumeMounts_Errors(t *testing.T) { updateRA: setComponentAndJobsVolumeMounts, expectedError: radixvalidators.ErrVolumeMountInvalidRequestsStorage, }, - "azureFile: not implemented": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Name: "some_name", - Path: "some_path", - AzureFile: &radixv1.RadixAzureFileVolumeMount{}, - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountTypeNotImplemented, - }, "emptyDir: valid": { volumeMounts: func() []radixv1.RadixVolumeMount { volumeMounts := []radixv1.RadixVolumeMount{ @@ -2237,10 +2204,8 @@ func Test_ValidateApplicationCanBeAppliedWithDNSAliases(t *testing.T) { otherAppName = "anyapp2" raEnv = "test" raComponentName = "app" - raPublicPort = 8080 someEnv = "dev" someComponentName = "component-abc" - somePort = 9090 alias1 = "alias1" alias2 = "alias2" ) diff --git a/pkg/apis/utils/storageclass.go b/pkg/apis/utils/persistentvolume.go similarity index 77% rename from pkg/apis/utils/storageclass.go rename to pkg/apis/utils/persistentvolume.go index 7f6ef12cf..61283f660 100644 --- a/pkg/apis/utils/storageclass.go +++ b/pkg/apis/utils/persistentvolume.go @@ -40,23 +40,23 @@ func EqualPersistentVolumeLists(list1, list2 *[]corev1.PersistentVolume) (bool, // EqualPersistentVolumes Compare two PersistentVolumes func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) (bool, error) { - pv1Copy, labels1, params1, mountOptions1 := getPersistentVolumeCopyWithCollections(pv1) - pv2Copy, labels2, params2, mountOptions2 := getPersistentVolumeCopyWithCollections(pv2) + pv1Copy, labels1, attribs1, mountOptions1 := getPersistentVolumeCopyWithCollections(pv1) + pv2Copy, labels2, attribs2, mountOptions2 := getPersistentVolumeCopyWithCollections(pv2) patchBytes, err := getPersistentVolumePatch(pv1Copy, pv2Copy) if err != nil { return false, err } if !EqualStringMaps(labels1, labels2) { - return false, nil // StorageClasses labels are not equal + return false, nil // PersistentVolume labels are not equal } - if !EqualStringMaps(params1, params2) { - return false, nil // StorageClasses parameters are not equal + if !EqualStringMaps(attribs1, attribs2) { + return false, nil // PersistentVolume parameters are not equal } if !EqualStringLists(mountOptions1, mountOptions2) { - return false, nil // StorageClass-es MountOptions are not equal + return false, nil // PersistentVolume-es MountOptions are not equal } if !kube.IsEmptyPatch(patchBytes) { - return false, nil // StorageClasses properties are not equal + return false, nil // PersistentVolume properties are not equal } return true, nil } @@ -67,13 +67,13 @@ func getPersistentVolumeCopyWithCollections(pv *corev1.PersistentVolume) (*corev // to avoid label order variations labels := pvCopy.ObjectMeta.Labels pvCopy.ObjectMeta.Labels = map[string]string{} - // to avoid Parameters order variations - scParams := pvCopy.Spec.Parameters // TODO nodeStorageSecretRef: name, ns - pvCopy.Parameters = map[string]string{} + // to avoid Attribs order variations + pvAttribs := pvCopy.Spec.CSI.VolumeAttributes + pvCopy.Spec.CSI.VolumeAttributes = map[string]string{} // to avoid MountOptions order variations mountOptions := pvCopy.Spec.MountOptions pvCopy.Spec.MountOptions = []string{} - return pvCopy, labels, scParams, mountOptions + return pvCopy, labels, pvAttribs, mountOptions } func getPersistentVolumePatch(pv1, pv2 *corev1.PersistentVolume) ([]byte, error) { From 6bf84c401075e18160718f66fcb35fa9d61e2d94 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Wed, 13 Nov 2024 16:26:25 +0100 Subject: [PATCH 03/68] Added blobfuse properties --- .../templates/radixapplication.yaml | 154 ++++++++++++------ json-schema/radixapplication.json | 136 ++++++++++------ pkg/apis/radix/v1/radixapptypes.go | 18 +- pkg/apis/radix/v1/zz_generated.deepcopy.go | 5 + 4 files changed, 206 insertions(+), 107 deletions(-) diff --git a/charts/radix-operator/templates/radixapplication.yaml b/charts/radix-operator/templates/radixapplication.yaml index 20133fde9..824b5a92f 100644 --- a/charts/radix-operator/templates/radixapplication.yaml +++ b/charts/radix-operator/templates/radixapplication.yaml @@ -913,7 +913,7 @@ spec: description: |- Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce @@ -921,8 +921,9 @@ spec: - "" type: string azureFile: - description: AzureFile settings for Azure File CSI - driver + description: |- + AzureFile settings for Azure File CSI driver + Deprecated. properties: accessMode: description: |- @@ -972,7 +973,7 @@ spec: description: |- Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -980,7 +981,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage - FUSE CSI driver + FUSE CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -1015,7 +1016,6 @@ spec: Storage FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -1023,6 +1023,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. + Applicable when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -1034,6 +1038,10 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. If not + defined - it will be configured in a secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -1110,7 +1118,7 @@ spec: gid: description: |- GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -1127,33 +1135,38 @@ spec: requestsStorage: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string skuName: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string storage: description: |- Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string type: description: |- Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - blob - azure-blob - - azure-file - "" type: string uid: description: |- UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Key Vault will be acquired + using Azure Workload Identity instead of using + a ClientID and Secret. + type: boolean required: - name - path @@ -1708,7 +1721,7 @@ spec: description: |- Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce @@ -1716,7 +1729,9 @@ spec: - "" type: string azureFile: - description: AzureFile settings for Azure File CSI driver + description: |- + AzureFile settings for Azure File CSI driver + Deprecated. properties: accessMode: description: |- @@ -1764,7 +1779,7 @@ spec: description: |- Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -1772,7 +1787,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage FUSE - CSI driver + CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -1806,7 +1821,6 @@ spec: FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -1814,6 +1828,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. Applicable + when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -1825,6 +1843,10 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. If not defined + - it will be configured in a secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -1898,7 +1920,7 @@ spec: gid: description: |- GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -1915,33 +1937,38 @@ spec: requestsStorage: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string skuName: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string storage: description: |- Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string type: description: |- Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - blob - azure-blob - - azure-file - "" type: string uid: description: |- UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Key Vault will be acquired using + Azure Workload Identity instead of using a ClientID + and Secret. + type: boolean required: - name - path @@ -2560,7 +2587,7 @@ spec: description: |- Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce @@ -2568,8 +2595,9 @@ spec: - "" type: string azureFile: - description: AzureFile settings for Azure File CSI - driver + description: |- + AzureFile settings for Azure File CSI driver + Deprecated. properties: accessMode: description: |- @@ -2619,7 +2647,7 @@ spec: description: |- Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -2627,7 +2655,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage - FUSE CSI driver + FUSE CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -2662,7 +2690,6 @@ spec: Storage FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -2670,6 +2697,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. + Applicable when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -2681,6 +2712,10 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. If not + defined - it will be configured in a secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -2757,7 +2792,7 @@ spec: gid: description: |- GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -2774,33 +2809,38 @@ spec: requestsStorage: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string skuName: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string storage: description: |- Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string type: description: |- Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - blob - azure-blob - - azure-file - "" type: string uid: description: |- UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Key Vault will be acquired + using Azure Workload Identity instead of using + a ClientID and Secret. + type: boolean required: - name - path @@ -3111,7 +3151,7 @@ spec: description: |- Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce @@ -3119,7 +3159,9 @@ spec: - "" type: string azureFile: - description: AzureFile settings for Azure File CSI driver + description: |- + AzureFile settings for Azure File CSI driver + Deprecated. properties: accessMode: description: |- @@ -3167,7 +3209,7 @@ spec: description: |- Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -3175,7 +3217,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage FUSE - CSI driver + CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -3209,7 +3251,6 @@ spec: FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -3217,6 +3258,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. Applicable + when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -3228,6 +3273,10 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. If not defined + - it will be configured in a secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -3301,7 +3350,7 @@ spec: gid: description: |- GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -3318,33 +3367,38 @@ spec: requestsStorage: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string skuName: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string storage: description: |- Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string type: description: |- Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - blob - azure-blob - - azure-file - "" type: string uid: description: |- UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Key Vault will be acquired using + Azure Workload Identity instead of using a ClientID + and Secret. + type: boolean required: - name - path diff --git a/json-schema/radixapplication.json b/json-schema/radixapplication.json index c4c1a3ff3..6eeb9ebad 100644 --- a/json-schema/radixapplication.json +++ b/json-schema/radixapplication.json @@ -882,7 +882,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -892,7 +892,7 @@ "type": "string" }, "azureFile": { - "description": "AzureFile settings for Azure File CSI driver", + "description": "AzureFile settings for Azure File CSI driver\nDeprecated.", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -937,7 +937,7 @@ "type": "object" }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -946,7 +946,7 @@ "type": "string" }, "blobFuse2": { - "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver", + "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver with the protocol fuse2", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -979,7 +979,6 @@ "description": "Holds protocols of BlobFuse2 Azure Storage FUSE driver. Default is fuse2.", "enum": [ "fuse2", - "nfs", "" ], "type": "string" @@ -988,6 +987,10 @@ "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", "type": "string" }, + "resourceGroup": { + "description": "ResourceGroup of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "skuName": { "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", "enum": [ @@ -999,6 +1002,10 @@ ], "type": "string" }, + "storageAccount": { + "description": "Name of a storage account. If not defined - it will be configured in a secret.", + "type": "string" + }, "streaming": { "description": "Configure Streaming mode. Used for blobfuse2.\nMore info: https://github.com/Azure/azure-storage-fuse/blob/main/STREAMING.md", "properties": { @@ -1080,7 +1087,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "name": { @@ -1095,30 +1102,33 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 instead.", "enum": [ "blob", "azure-blob", - "azure-file", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", "type": "string" + }, + "useAzureIdentity": { + "description": "UseAzureIdentity defines that credentials for accessing Azure Key Vault will be acquired using Azure Workload Identity instead of using a ClientID and Secret.", + "type": "boolean" } }, "required": [ @@ -1697,7 +1707,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -1707,7 +1717,7 @@ "type": "string" }, "azureFile": { - "description": "AzureFile settings for Azure File CSI driver", + "description": "AzureFile settings for Azure File CSI driver\nDeprecated.", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -1752,7 +1762,7 @@ "type": "object" }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -1761,7 +1771,7 @@ "type": "string" }, "blobFuse2": { - "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver", + "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver with the protocol fuse2", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -1794,7 +1804,6 @@ "description": "Holds protocols of BlobFuse2 Azure Storage FUSE driver. Default is fuse2.", "enum": [ "fuse2", - "nfs", "" ], "type": "string" @@ -1803,6 +1812,10 @@ "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", "type": "string" }, + "resourceGroup": { + "description": "ResourceGroup of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "skuName": { "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", "enum": [ @@ -1814,6 +1827,10 @@ ], "type": "string" }, + "storageAccount": { + "description": "Name of a storage account. If not defined - it will be configured in a secret.", + "type": "string" + }, "streaming": { "description": "Configure Streaming mode. Used for blobfuse2.\nMore info: https://github.com/Azure/azure-storage-fuse/blob/main/STREAMING.md", "properties": { @@ -1895,7 +1912,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "name": { @@ -1910,30 +1927,33 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 instead.", "enum": [ "blob", "azure-blob", - "azure-file", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", "type": "string" + }, + "useAzureIdentity": { + "description": "UseAzureIdentity defines that credentials for accessing Azure Key Vault will be acquired using Azure Workload Identity instead of using a ClientID and Secret.", + "type": "boolean" } }, "required": [ @@ -2588,7 +2608,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -2598,7 +2618,7 @@ "type": "string" }, "azureFile": { - "description": "AzureFile settings for Azure File CSI driver", + "description": "AzureFile settings for Azure File CSI driver\nDeprecated.", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -2643,7 +2663,7 @@ "type": "object" }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -2652,7 +2672,7 @@ "type": "string" }, "blobFuse2": { - "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver", + "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver with the protocol fuse2", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -2685,7 +2705,6 @@ "description": "Holds protocols of BlobFuse2 Azure Storage FUSE driver. Default is fuse2.", "enum": [ "fuse2", - "nfs", "" ], "type": "string" @@ -2694,6 +2713,10 @@ "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", "type": "string" }, + "resourceGroup": { + "description": "ResourceGroup of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "skuName": { "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", "enum": [ @@ -2705,6 +2728,10 @@ ], "type": "string" }, + "storageAccount": { + "description": "Name of a storage account. If not defined - it will be configured in a secret.", + "type": "string" + }, "streaming": { "description": "Configure Streaming mode. Used for blobfuse2.\nMore info: https://github.com/Azure/azure-storage-fuse/blob/main/STREAMING.md", "properties": { @@ -2786,7 +2813,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "name": { @@ -2801,30 +2828,33 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 instead.", "enum": [ "blob", "azure-blob", - "azure-file", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", "type": "string" + }, + "useAzureIdentity": { + "description": "UseAzureIdentity defines that credentials for accessing Azure Key Vault will be acquired using Azure Workload Identity instead of using a ClientID and Secret.", + "type": "boolean" } }, "required": [ @@ -3148,7 +3178,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -3158,7 +3188,7 @@ "type": "string" }, "azureFile": { - "description": "AzureFile settings for Azure File CSI driver", + "description": "AzureFile settings for Azure File CSI driver\nDeprecated.", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -3203,7 +3233,7 @@ "type": "object" }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -3212,7 +3242,7 @@ "type": "string" }, "blobFuse2": { - "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver", + "description": "BlobFuse2 settings for Azure Storage FUSE CSI driver with the protocol fuse2", "properties": { "accessMode": { "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", @@ -3245,7 +3275,6 @@ "description": "Holds protocols of BlobFuse2 Azure Storage FUSE driver. Default is fuse2.", "enum": [ "fuse2", - "nfs", "" ], "type": "string" @@ -3254,6 +3283,10 @@ "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", "type": "string" }, + "resourceGroup": { + "description": "ResourceGroup of a storage account. Applicable when using a workload identity.", + "type": "string" + }, "skuName": { "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", "enum": [ @@ -3265,6 +3298,10 @@ ], "type": "string" }, + "storageAccount": { + "description": "Name of a storage account. If not defined - it will be configured in a secret.", + "type": "string" + }, "streaming": { "description": "Configure Streaming mode. Used for blobfuse2.\nMore info: https://github.com/Azure/azure-storage-fuse/blob/main/STREAMING.md", "properties": { @@ -3346,7 +3383,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "name": { @@ -3361,30 +3398,33 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 instead.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 instead.", "enum": [ "blob", "azure-blob", - "azure-file", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 or AzureFile instead.", + "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", "type": "string" + }, + "useAzureIdentity": { + "description": "UseAzureIdentity defines that credentials for accessing Azure Key Vault will be acquired using Azure Workload Identity instead of using a ClientID and Secret.", + "type": "boolean" } }, "required": [ diff --git a/pkg/apis/radix/v1/radixapptypes.go b/pkg/apis/radix/v1/radixapptypes.go index c8bfef50d..77b5128a0 100644 --- a/pkg/apis/radix/v1/radixapptypes.go +++ b/pkg/apis/radix/v1/radixapptypes.go @@ -884,11 +884,15 @@ type RadixVolumeMount struct { // +optional BindingMode string `json:"bindingMode,omitempty"` // Volume binding mode. Available values: Immediate (default), WaitForFirstConsumer. https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode - // BlobFuse2 settings for Azure Storage FUSE CSI driver + // UseAzureIdentity defines that credentials for accessing Azure Key Vault will be acquired using Azure Workload Identity instead of using a ClientID and Secret. + // +optional + UseAzureIdentity *bool `json:"useAzureIdentity,omitempty"` + + // BlobFuse2 settings for Azure Storage FUSE CSI driver with the protocol fuse2 BlobFuse2 *RadixBlobFuse2VolumeMount `json:"blobFuse2,omitempty"` // AzureFile settings for Azure File CSI driver - // Deprecated, use BlobFuse2 instead. + // Deprecated. AzureFile *RadixAzureFileVolumeMount `json:"azureFile,omitempty"` // EmptyDir settings for EmptyDir volume @@ -974,17 +978,13 @@ type RadixBlobFuse2VolumeMount struct { // +optional Streaming *RadixVolumeMountStreaming `json:"streaming,omitempty"` // Optional. Streaming configuration. Used for blobfuse2. - // Name of a storage account. If not defined - it will be configured in a secret. - // +optional - StorageAccount string - - // ClientID of a service principal. Applicable when using a workload identity. + // Name of a storage account. It is mandatory when using a workload identity. It is optional when using Access Key, if it is not defined, it will be configured in a secret. // +optional - ClientId string + StorageAccount string `json:"storageAccount,omitempty"` // ResourceGroup of a storage account. Applicable when using a workload identity. // +optional - ResourceGroup string + ResourceGroup string `json:"resourceGroup,omitempty"` } // RadixAzureFileVolumeMount defines an external storage resource, configured to use Azure File with CSI driver. diff --git a/pkg/apis/radix/v1/zz_generated.deepcopy.go b/pkg/apis/radix/v1/zz_generated.deepcopy.go index fa71c6a19..bfa279278 100644 --- a/pkg/apis/radix/v1/zz_generated.deepcopy.go +++ b/pkg/apis/radix/v1/zz_generated.deepcopy.go @@ -2952,6 +2952,11 @@ func (in *RadixSecretRefs) DeepCopy() *RadixSecretRefs { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RadixVolumeMount) DeepCopyInto(out *RadixVolumeMount) { *out = *in + if in.UseAzureIdentity != nil { + in, out := &in.UseAzureIdentity, &out.UseAzureIdentity + *out = new(bool) + **out = **in + } if in.BlobFuse2 != nil { in, out := &in.BlobFuse2, &out.BlobFuse2 *out = new(RadixBlobFuse2VolumeMount) From 2a0e905d94dd4ec83ac7a358ad17d9f251cab5ff Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Wed, 13 Nov 2024 16:52:01 +0100 Subject: [PATCH 04/68] Added blobfuse properties --- pkg/apis/deployment/volumemount.go | 47 +++++++++++++++++-------- pkg/apis/deployment/volumemount_test.go | 3 +- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index 7be5e683d..f533b08fe 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -55,6 +55,7 @@ const ( csiVolumeMountAttributeStorageAccount = "storageAccount" csiVolumeMountAttributeClientID = "clientID" csiVolumeMountAttributeResourceGroup = "resourcegroup" + azureCsiDriverVolumeHandle = "azure-csi-driver-volume-handle" csiSecretStoreDriver = "secrets-store.csi.k8s.io" csiVolumeSourceVolumeAttrSecretProviderClassName = "secretProviderClass" @@ -63,7 +64,7 @@ const ( volumeNameMaxLength = 63 ) -// These are valid storage class provisioners +// These are valid volume mount provisioners const ( // provisionerBlobCsiAzure Use of azure/csi driver for blob in Azure storage account provisionerBlobCsiAzure string = "blob.csi.azure.com" @@ -555,7 +556,9 @@ func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appNa return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) } -func populateCsiAzureVolumeMount(persistentVolume *corev1.PersistentVolume, appName string, volumeRootMount string, namespace string, componentName string, pvName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, provisioner string) error { +func populateCsiAzureVolumeMount(persistentVolume *corev1.PersistentVolume, appName string, volumeRootMount string, namespace string, componentName string, pvName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, provisioner string, identity *radixv1.Identity) error { + identityClientId := getIdentityClientId(identity) + useAzureIdentity := getUseAzureIdentity(identity, radixVolumeMount) persistentVolume.ObjectMeta.Name = pvName persistentVolume.ObjectMeta.Labels = getCsiAzurePersistentVolumeLabels(appName, namespace, componentName, radixVolumeMount) persistentVolume.ObjectMeta.Annotations = getCsiAzurePersistentVolumeAnnotations(provisioner, secretName, namespace) @@ -565,15 +568,26 @@ func populateCsiAzureVolumeMount(persistentVolume *corev1.PersistentVolume, appN } persistentVolume.Spec.MountOptions = mountOptions persistentVolume.Spec.CSI = &corev1.CSIPersistentVolumeSource{ - VolumeHandle: "azure-csi-driver-volume-handle", - Driver: "blob.csi.azure.com", - VolumeAttributes: getCsiAzurePersistentVolumeAttributes(radixVolumeMount), + VolumeHandle: azureCsiDriverVolumeHandle, + Driver: provisionerBlobCsiAzure, + VolumeAttributes: getCsiAzurePersistentVolumeAttributes(radixVolumeMount, useAzureIdentity, identityClientId), } persistentVolume.Spec.CSI.NodePublishSecretRef = &corev1.SecretReference{Name: secretName, Namespace: namespace} // TODO - remove for Workload identity persistentVolume.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimRetain. PersistentVolumeReclaimPolicy deletes volume on unmount. return nil } +func getUseAzureIdentity(identity *radixv1.Identity, radixVolumeMount *radixv1.RadixVolumeMount) bool { + return len(getIdentityClientId(identity)) > 0 && radixVolumeMount.UseAzureIdentity != nil && *radixVolumeMount.UseAzureIdentity +} + +func getIdentityClientId(identity *radixv1.Identity) string { + if identity != nil && identity.Azure != nil && len(identity.Azure.ClientId) > 0 { + return identity.Azure.ClientId + } + return "" +} + func getCsiAzurePersistentVolumeAnnotations(provisioner, secretName, namespace string) map[string]string { return map[string]string{ "pv.kubernetes.io/provisioned-by: blob.csi.azure.com": provisioner, @@ -592,7 +606,7 @@ func getCsiAzurePersistentVolumeLabels(appName, namespace, componentName string, } } -func getCsiAzurePersistentVolumeAttributes(radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { +func getCsiAzurePersistentVolumeAttributes(radixVolumeMount *radixv1.RadixVolumeMount, useAzureIdentity bool, clientId string) map[string]string { attributes := make(map[string]string) switch GetCsiAzureVolumeMountType(radixVolumeMount) { case radixv1.MountTypeBlobFuse2FuseCsiAzure: @@ -601,9 +615,13 @@ func getCsiAzurePersistentVolumeAttributes(radixVolumeMount *radixv1.RadixVolume case radixv1.MountTypeBlobFuse2Fuse2CsiAzure: attributes[csiVolumeMountAttributeContainerName] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) attributes[csiVolumeMountAttributeProtocol] = csiPersistentVolumeProtocolParameterFuse2 - attributes[csiVolumeMountAttributeStorageAccount] = radixVolumeMount.BlobFuse2.StorageAccount - attributes[csiVolumeMountAttributeClientID] = radixVolumeMount.BlobFuse2.ClientId - attributes[csiVolumeMountAttributeResourceGroup] = radixVolumeMount.BlobFuse2.ResourceGroup + if len(radixVolumeMount.BlobFuse2.StorageAccount) > 0 { + attributes[csiVolumeMountAttributeStorageAccount] = radixVolumeMount.BlobFuse2.StorageAccount + } + if useAzureIdentity { + attributes[csiVolumeMountAttributeClientID] = clientId + attributes[csiVolumeMountAttributeResourceGroup] = radixVolumeMount.BlobFuse2.ResourceGroup + } } return attributes } @@ -790,7 +808,7 @@ func knownCSIDriver(csiPersistentVolumeSource *corev1.CSIPersistentVolumeSource) } // createOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims, PersistentVolume -func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Context, desiredDeployment *appsv1.Deployment) error { +func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Context, desiredDeployment *appsv1.Deployment, deployComponent radixv1.RadixDeployComponent) error { namespace := deploy.radixDeployment.GetNamespace() appName := deploy.radixDeployment.Spec.AppName componentName := desiredDeployment.ObjectMeta.Name @@ -812,6 +830,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont if err != nil { return err } + identity := deployComponent.GetIdentity() for _, volume := range desiredDeployment.Spec.Template.Spec.Volumes { if volume.PersistentVolumeClaim == nil { continue @@ -820,7 +839,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont if !existsRadixVolumeMount { return fmt.Errorf("not found Radix volume mount for desired volume %s", volume.Name) } - pv, pvIsCreated, err := deploy.getOrCreateCsiAzurePersistentVolume(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, volume.Name, scMap) + pv, pvIsCreated, err := deploy.getOrCreateCsiAzurePersistentVolume(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, volume.Name, scMap, identity) if err != nil { return err } @@ -909,7 +928,7 @@ func (deploy *Deployment) createCsiAzurePersistentVolumeClaim(ctx context.Contex return deploy.createPersistentVolumeClaim(ctx, appName, namespace, componentName, persistentVolumeClaimName, pv.Name, radixVolumeMount) } -func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Context, appName, volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, volumeName string, pvMap map[string]*corev1.PersistentVolume) (*corev1.PersistentVolume, bool, error) { +func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Context, appName, volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, volumeName string, pvMap map[string]*corev1.PersistentVolume, identity *radixv1.Identity) (*corev1.PersistentVolume, bool, error) { var volumeMountProvisioner, foundProvisioner = getPersistentVolumeProvisionerByVolumeMountType(radixVolumeMount) if !foundProvisioner { return nil, false, fmt.Errorf("not found Storage Class provisioner for volume mount type %s", string(GetCsiAzureVolumeMountType(radixVolumeMount))) @@ -918,7 +937,7 @@ func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Contex csiVolumeSecretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, radixVolumeMount.Name) if existingPersistentVolume, exists := pvMap[pvName]; exists { desiredPersistentVolume := existingPersistentVolume.DeepCopy() - err := populateCsiAzureVolumeMount(desiredPersistentVolume, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) + err := populateCsiAzureVolumeMount(desiredPersistentVolume, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner, identity) if err != nil { return nil, false, err } @@ -935,7 +954,7 @@ func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Contex log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) pv := &corev1.PersistentVolume{} - err := populateCsiAzureVolumeMount(pv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner) + err := populateCsiAzureVolumeMount(pv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner, nil) if err != nil { return nil, false, err } diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 0e80c026a..95278ffb1 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -959,7 +959,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { desiredDeployment := getDesiredDeployment(componentName, scenario.volumes) // action - err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), desiredDeployment) + deployComponent := deployment.radixDeployment.Spec.Components[0] + err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), desiredDeployment, deployComponent) assert.Nil(t, err) existingPvcs, existingScs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment) From 9ba8425f255d06747479743be21c2c87c3d1d614 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 15 Nov 2024 15:40:30 +0100 Subject: [PATCH 05/68] Added blobfuse properties --- pkg/apis/deployment/volumemount.go | 100 ++++++++++++++--------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index f533b08fe..ec593eeef 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -74,18 +74,6 @@ var ( csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}} ) -// getPersistentVolumeProvisionerByVolumeMountType convert volume mount type to Storage Class provisioner -func getPersistentVolumeProvisionerByVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) (string, bool) { - if radixVolumeMount.BlobFuse2 != nil { - return provisionerBlobCsiAzure, true - } - switch radixVolumeMount.Type { - case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeBlobFuse2Fuse2CsiAzure: - return provisionerBlobCsiAzure, true - } - return "", false -} - // isKnownCsiAzureVolumeMount Supported volume mount type CSI Azure Blob volume func isKnownCsiAzureVolumeMount(volumeMount string) bool { switch volumeMount { @@ -193,6 +181,8 @@ func GetCsiAzureVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) radi func getCsiRadixVolumeTypeIdForName(radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { if radixVolumeMount.BlobFuse2 != nil { switch radixVolumeMount.BlobFuse2.Protocol { + case radixv1.BlobFuse2ProtocolFuse2, "": + return "csi-blobfuse2-fuse2", nil default: return "", fmt.Errorf("unknown blobfuse2 protocol %s", radixVolumeMount.BlobFuse2.Protocol) } @@ -533,7 +523,6 @@ func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appNa if err != nil { requestsVolumeMountSize = resource.MustParse("1Mi") } - volumeAccessMode := getVolumeAccessMode(getRadixBlobFuse2VolumeMountAccessMode(radixVolumeMount)) pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, @@ -546,7 +535,7 @@ func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appNa }, }, Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{volumeAccessMode}, + AccessModes: []corev1.PersistentVolumeAccessMode{getVolumeMountAccessMode(radixVolumeMount)}, Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceStorage: requestsVolumeMountSize}, // it seems correct number is not needed for CSI driver }, @@ -556,24 +545,34 @@ func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appNa return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) } -func populateCsiAzureVolumeMount(persistentVolume *corev1.PersistentVolume, appName string, volumeRootMount string, namespace string, componentName string, pvName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, provisioner string, identity *radixv1.Identity) error { +func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, appName string, volumeRootMount string, namespace string, componentName string, pvName string, radixVolumeMount *radixv1.RadixVolumeMount, identity *radixv1.Identity) error { identityClientId := getIdentityClientId(identity) useAzureIdentity := getUseAzureIdentity(identity, radixVolumeMount) + csiVolumeCredSecretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, radixVolumeMount.Name) persistentVolume.ObjectMeta.Name = pvName persistentVolume.ObjectMeta.Labels = getCsiAzurePersistentVolumeLabels(appName, namespace, componentName, radixVolumeMount) - persistentVolume.ObjectMeta.Annotations = getCsiAzurePersistentVolumeAnnotations(provisioner, secretName, namespace) + persistentVolume.ObjectMeta.Annotations = getCsiAzurePersistentVolumeAnnotations(csiVolumeCredSecretName, namespace, useAzureIdentity) mountOptions, err := getCsiAzurePersistentVolumeMountOptions(volumeRootMount, namespace, componentName, radixVolumeMount) if err != nil { return err } persistentVolume.Spec.MountOptions = mountOptions + persistentVolume.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{getVolumeMountAccessMode(radixVolumeMount)} + persistentVolume.Spec.ClaimRef = &corev1.ObjectReference{ + APIVersion: "v1", + Kind: persistentVolumeClaimKind, + Namespace: namespace, + Name: pvName, + } persistentVolume.Spec.CSI = &corev1.CSIPersistentVolumeSource{ - VolumeHandle: azureCsiDriverVolumeHandle, Driver: provisionerBlobCsiAzure, - VolumeAttributes: getCsiAzurePersistentVolumeAttributes(radixVolumeMount, useAzureIdentity, identityClientId), + VolumeHandle: azureCsiDriverVolumeHandle, + VolumeAttributes: getCsiAzurePersistentVolumeAttributes(radixVolumeMount, pvName, namespace, useAzureIdentity, identityClientId), + } + if !useAzureIdentity { + persistentVolume.Spec.CSI.NodeStageSecretRef = &corev1.SecretReference{Name: csiVolumeCredSecretName, Namespace: namespace} } - persistentVolume.Spec.CSI.NodePublishSecretRef = &corev1.SecretReference{Name: secretName, Namespace: namespace} // TODO - remove for Workload identity - persistentVolume.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimRetain. PersistentVolumeReclaimPolicy deletes volume on unmount. + persistentVolume.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimRetain. PersistentVolumeReclaimPolicy deletes volume on unmount. return nil } @@ -588,12 +587,15 @@ func getIdentityClientId(identity *radixv1.Identity) string { return "" } -func getCsiAzurePersistentVolumeAnnotations(provisioner, secretName, namespace string) map[string]string { - return map[string]string{ - "pv.kubernetes.io/provisioned-by: blob.csi.azure.com": provisioner, - "volume.kubernetes.io/provisioner-deletion-secret-name": secretName, - "volume.kubernetes.io/provisioner-deletion-secret-namespace": namespace, +func getCsiAzurePersistentVolumeAnnotations(csiVolumeCredSecretName, namespace string, useAzureIdentity bool) map[string]string { + annotationsMap := map[string]string{ + "pv.kubernetes.io/provisioned-by": provisionerBlobCsiAzure, + } + if !useAzureIdentity { + annotationsMap["volume.kubernetes.io/provisioner-deletion-secret-name"] = csiVolumeCredSecretName + annotationsMap["volume.kubernetes.io/provisioner-deletion-secret-namespace"] = namespace } + return annotationsMap } func getCsiAzurePersistentVolumeLabels(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { @@ -601,12 +603,11 @@ func getCsiAzurePersistentVolumeLabels(appName, namespace, componentName string, kube.RadixAppLabel: appName, kube.RadixNamespace: namespace, kube.RadixComponentLabel: componentName, - kube.RadixMountTypeLabel: string(GetCsiAzureVolumeMountType(radixVolumeMount)), kube.RadixVolumeMountNameLabel: radixVolumeMount.Name, } } -func getCsiAzurePersistentVolumeAttributes(radixVolumeMount *radixv1.RadixVolumeMount, useAzureIdentity bool, clientId string) map[string]string { +func getCsiAzurePersistentVolumeAttributes(radixVolumeMount *radixv1.RadixVolumeMount, pvName, namespace string, useAzureIdentity bool, clientId string) map[string]string { attributes := make(map[string]string) switch GetCsiAzureVolumeMountType(radixVolumeMount) { case radixv1.MountTypeBlobFuse2FuseCsiAzure: @@ -623,6 +624,13 @@ func getCsiAzurePersistentVolumeAttributes(radixVolumeMount *radixv1.RadixVolume attributes[csiVolumeMountAttributeResourceGroup] = radixVolumeMount.BlobFuse2.ResourceGroup } } + attributes["csi.storage.k8s.io/pv/name"] = pvName + attributes["csi.storage.k8s.io/pvc/name"] = pvName + attributes["csi.storage.k8s.io/pvc/namespace"] = namespace + if !useAzureIdentity { + attributes["secretnamespace"] = namespace + } + // ? storage.kubernetes.io/csiProvisionerIdentity: 1731647415428-2825-blob.csi.azure.com return attributes } @@ -655,7 +663,7 @@ func getCsiAzurePersistentVolumeMountOptionsForAzureBlob(tmpPath string, radixVo mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", csiPersistentVolumeUidMountOption, uid)) } } - if getRadixBlobFuse2VolumeMountAccessMode(radixVolumeMount) == string(corev1.ReadOnlyMany) { + if getVolumeMountAccessMode(radixVolumeMount) == corev1.ReadOnlyMany { mountOptions = append(mountOptions, "-o ro") } if radixVolumeMount.BlobFuse2 != nil { @@ -692,11 +700,20 @@ func getStreamingMountOptions(streaming *radixv1.RadixVolumeMountStreaming) []st return mountOptions } -func getRadixBlobFuse2VolumeMountAccessMode(radixVolumeMount *radixv1.RadixVolumeMount) string { +func getVolumeMountAccessMode(radixVolumeMount *radixv1.RadixVolumeMount) corev1.PersistentVolumeAccessMode { + accessMode := radixVolumeMount.AccessMode if radixVolumeMount.BlobFuse2 != nil { - return radixVolumeMount.BlobFuse2.AccessMode + accessMode = radixVolumeMount.BlobFuse2.AccessMode + } + switch strings.ToLower(accessMode) { + case strings.ToLower(string(corev1.ReadWriteOnce)): + return corev1.ReadWriteOnce + case strings.ToLower(string(corev1.ReadWriteMany)): + return corev1.ReadWriteMany + case strings.ToLower(string(corev1.ReadWriteOncePod)): + return corev1.ReadWriteOncePod } - return radixVolumeMount.AccessMode + return corev1.ReadOnlyMany // default access mode } func getRadixBlobFuse2VolumeMountUid(radixVolumeMount *radixv1.RadixVolumeMount) string { @@ -929,15 +946,10 @@ func (deploy *Deployment) createCsiAzurePersistentVolumeClaim(ctx context.Contex } func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Context, appName, volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, volumeName string, pvMap map[string]*corev1.PersistentVolume, identity *radixv1.Identity) (*corev1.PersistentVolume, bool, error) { - var volumeMountProvisioner, foundProvisioner = getPersistentVolumeProvisionerByVolumeMountType(radixVolumeMount) - if !foundProvisioner { - return nil, false, fmt.Errorf("not found Storage Class provisioner for volume mount type %s", string(GetCsiAzureVolumeMountType(radixVolumeMount))) - } pvName := GetCsiAzurePersistentVolumeName(namespace, volumeName) - csiVolumeSecretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, radixVolumeMount.Name) if existingPersistentVolume, exists := pvMap[pvName]; exists { desiredPersistentVolume := existingPersistentVolume.DeepCopy() - err := populateCsiAzureVolumeMount(desiredPersistentVolume, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner, identity) + err := populateCsiAzurePersistentVolume(desiredPersistentVolume, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, identity) if err != nil { return nil, false, err } @@ -954,7 +966,7 @@ func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Contex log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) pv := &corev1.PersistentVolume{} - err := populateCsiAzureVolumeMount(pv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, csiVolumeSecretName, volumeMountProvisioner, nil) + err := populateCsiAzurePersistentVolume(pv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, nil) if err != nil { return nil, false, err } @@ -995,18 +1007,6 @@ func findCsiAzureVolumeForComponent(volumeMountMap map[string]*radixv1.RadixVolu return true } -func getVolumeAccessMode(modeValue string) corev1.PersistentVolumeAccessMode { - switch strings.ToLower(modeValue) { - case strings.ToLower(string(corev1.ReadWriteOnce)): - return corev1.ReadWriteOnce - case strings.ToLower(string(corev1.ReadWriteMany)): - return corev1.ReadWriteMany - case strings.ToLower(string(corev1.ReadWriteOncePod)): - return corev1.ReadWriteOncePod - } - return corev1.ReadOnlyMany // default access mode -} - func sortPvcsByCreatedTimestampDesc(persistentVolumeClaims []corev1.PersistentVolumeClaim) []corev1.PersistentVolumeClaim { sort.SliceStable(persistentVolumeClaims, func(i, j int) bool { return (persistentVolumeClaims)[j].ObjectMeta.CreationTimestamp.Before(&(persistentVolumeClaims)[i].ObjectMeta.CreationTimestamp) From 2ccb43ffbc16850c708a65a2783f12f4aca57946 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 25 Nov 2024 16:38:42 +0100 Subject: [PATCH 06/68] Added blobfuse properties --- pkg/apis/deployment/kubedeployment.go | 2 +- pkg/apis/deployment/secretrefs.go | 12 ++++---- pkg/apis/deployment/volumemount.go | 41 ++++++++++++------------- pkg/apis/deployment/volumemount_test.go | 2 +- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index d7084319c..e29015279 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -41,7 +41,7 @@ func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployCompone } } - err = deploy.createOrUpdateCsiAzureVolumeResources(ctx, desiredDeployment) + err = deploy.createOrUpdateCsiAzureVolumeResources(ctx, desiredDeployment, deployComponent) if err != nil { return err } diff --git a/pkg/apis/deployment/secretrefs.go b/pkg/apis/deployment/secretrefs.go index 5fa0fbb36..29b196a3c 100644 --- a/pkg/apis/deployment/secretrefs.go +++ b/pkg/apis/deployment/secretrefs.go @@ -30,9 +30,9 @@ func (deploy *Deployment) createSecretRefs(ctx context.Context, namespace string return nil, err } if credsSecret != nil && !isOwnerReference(credsSecret.ObjectMeta, secretProviderClass.ObjectMeta) { - credsSecret.ObjectMeta.OwnerReferences = append(credsSecret.ObjectMeta.OwnerReferences, getOwnerReferenceOfSecretProviderClass(secretProviderClass)) - _, err = deploy.kubeutil.ApplySecret(ctx, namespace, credsSecret) //nolint:staticcheck // must be updated to use UpdateSecret or CreateSecret - if err != nil { + updatedCredsSecret := credsSecret.DeepCopy() + updatedCredsSecret.ObjectMeta.OwnerReferences = append(credsSecret.ObjectMeta.OwnerReferences, getOwnerReferenceOfSecretProviderClass(secretProviderClass)) + if _, err = deploy.kubeutil.UpdateSecret(ctx, credsSecret, updatedCredsSecret); err != nil { return nil, err } } @@ -62,7 +62,7 @@ func (deploy *Deployment) getAzureKeyVaultCredsSecret(ctx context.Context, names func (deploy *Deployment) createAzureKeyVaultSecretProviderClassForRadixDeployment(ctx context.Context, namespace string, appName string, radixDeployComponentName string, azureKeyVault radixv1.RadixAzureKeyVault) (*secretsstorev1.SecretProviderClass, error) { radixDeploymentName := deploy.radixDeployment.GetName() tenantId := deploy.config.DeploymentSyncer.TenantID - identity := getIdentityFromRadixCommonDeployComponent(deploy, radixDeployComponentName) + identity := deploy.getIdentityFromRadixCommonDeployComponent(radixDeployComponentName) secretProviderClass, err := kube.BuildAzureKeyVaultSecretProviderClass(tenantId, appName, radixDeploymentName, radixDeployComponentName, azureKeyVault, identity) if err != nil { return nil, err @@ -73,7 +73,7 @@ func (deploy *Deployment) createAzureKeyVaultSecretProviderClassForRadixDeployme return deploy.kubeutil.CreateSecretProviderClass(ctx, namespace, secretProviderClass) } -func getIdentityFromRadixCommonDeployComponent(deploy *Deployment, radixDeployComponentName string) *radixv1.Identity { +func (deploy *Deployment) getIdentityFromRadixCommonDeployComponent(radixDeployComponentName string) *radixv1.Identity { if radixDeployComponent := deploy.radixDeployment.GetComponentByName(radixDeployComponentName); radixDeployComponent != nil { return radixDeployComponent.GetIdentity() } @@ -89,7 +89,7 @@ func (deploy *Deployment) getOrCreateAzureKeyVaultCredsSecret(ctx context.Contex if err != nil { if errors.IsNotFound(err) { secret = buildAzureKeyVaultCredentialsSecret(appName, componentName, secretName, azKeyVaultName) - return deploy.kubeutil.ApplySecret(ctx, namespace, secret) //nolint:staticcheck // must be updated to use UpdateSecret or CreateSecret + return deploy.kubeutil.CreateSecret(ctx, namespace, secret) } return nil, err } diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index ec593eeef..0cb498ec2 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -401,8 +401,7 @@ func createCsiAzurePersistentVolumeClaimName(componentName string, radixVolumeMo return fmt.Sprintf(csiPersistentVolumeClaimNameTemplate, volumeName, strings.ToLower(commonUtils.RandString(5))), nil // volumeName: --- } -// GetCsiAzurePersistentVolumeName hold a name of CSI volume storage class -func GetCsiAzurePersistentVolumeName(namespace, volumeName string) string { +func getCsiAzurePersistentVolumeName(namespace, volumeName string) string { return fmt.Sprintf(csiPersistentVolumeNameTemplate, namespace, volumeName) // volumeName: --- } @@ -545,9 +544,9 @@ func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appNa return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) } -func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, appName string, volumeRootMount string, namespace string, componentName string, pvName string, radixVolumeMount *radixv1.RadixVolumeMount, identity *radixv1.Identity) error { +func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, appName, volumeRootMount, namespace, componentName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount, identity *radixv1.Identity) error { identityClientId := getIdentityClientId(identity) - useAzureIdentity := getUseAzureIdentity(identity, radixVolumeMount) + useAzureIdentity := getUseAzureIdentity(identity, radixVolumeMount.UseAzureIdentity) csiVolumeCredSecretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, radixVolumeMount.Name) persistentVolume.ObjectMeta.Name = pvName persistentVolume.ObjectMeta.Labels = getCsiAzurePersistentVolumeLabels(appName, namespace, componentName, radixVolumeMount) @@ -576,8 +575,8 @@ func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, return nil } -func getUseAzureIdentity(identity *radixv1.Identity, radixVolumeMount *radixv1.RadixVolumeMount) bool { - return len(getIdentityClientId(identity)) > 0 && radixVolumeMount.UseAzureIdentity != nil && *radixVolumeMount.UseAzureIdentity +func getUseAzureIdentity(identity *radixv1.Identity, useAzureIdentity *bool) bool { + return len(getIdentityClientId(identity)) > 0 && useAzureIdentity != nil && *useAzureIdentity } func getIdentityClientId(identity *radixv1.Identity) string { @@ -825,12 +824,12 @@ func knownCSIDriver(csiPersistentVolumeSource *corev1.CSIPersistentVolumeSource) } // createOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims, PersistentVolume -func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Context, desiredDeployment *appsv1.Deployment, deployComponent radixv1.RadixDeployComponent) error { +func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Context, desiredDeployment *appsv1.Deployment, deployComponent radixv1.RadixCommonDeployComponent) error { namespace := deploy.radixDeployment.GetNamespace() appName := deploy.radixDeployment.Spec.AppName componentName := desiredDeployment.ObjectMeta.Name volumeRootMount := "/tmp" // TODO: add to environment variable, so this volume can be mounted to external disk - scList, err := deploy.getCsiAzurePersistentVolume(ctx, namespace, componentName) + pvList, err := deploy.getCsiAzurePersistentVolume(ctx, namespace, componentName) if err != nil { return err } @@ -839,7 +838,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont return err } - scMap := utils.GetPersistentVolumeMap(&scList.Items) + pvMap := utils.GetPersistentVolumeMap(&pvList.Items) pvcMap := utils.GetPersistentVolumeClaimMap(&pvcList.Items) radixVolumeMountMap := deploy.getRadixVolumeMountMapByCsiAzureVolumeMountName(componentName) var actualPersistentVolumeNames []string @@ -856,7 +855,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont if !existsRadixVolumeMount { return fmt.Errorf("not found Radix volume mount for desired volume %s", volume.Name) } - pv, pvIsCreated, err := deploy.getOrCreateCsiAzurePersistentVolume(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, volume.Name, scMap, identity) + pv, pvIsCreated, err := deploy.getOrCreateCsiAzurePersistentVolume(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, volume.Name, pvMap, identity) if err != nil { return err } @@ -946,31 +945,31 @@ func (deploy *Deployment) createCsiAzurePersistentVolumeClaim(ctx context.Contex } func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Context, appName, volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, volumeName string, pvMap map[string]*corev1.PersistentVolume, identity *radixv1.Identity) (*corev1.PersistentVolume, bool, error) { - pvName := GetCsiAzurePersistentVolumeName(namespace, volumeName) - if existingPersistentVolume, exists := pvMap[pvName]; exists { - desiredPersistentVolume := existingPersistentVolume.DeepCopy() - err := populateCsiAzurePersistentVolume(desiredPersistentVolume, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, identity) + pvName := getCsiAzurePersistentVolumeName(namespace, volumeName) + if existingPv, exists := pvMap[pvName]; exists { + desiredPv := existingPv.DeepCopy() + err := populateCsiAzurePersistentVolume(desiredPv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, identity) if err != nil { return nil, false, err } - if equal, err := utils.EqualPersistentVolumes(existingPersistentVolume, desiredPersistentVolume); equal || err != nil { - return existingPersistentVolume, false, err + if equal, err := utils.EqualPersistentVolumes(existingPv, desiredPv); equal || err != nil { + return existingPv, false, err } - log.Ctx(ctx).Info().Msgf("Delete PersistentVolume %s in namespace %s", existingPersistentVolume.Name, namespace) - err = deploy.deleteCsiAzurePersistentVolumes(ctx, existingPersistentVolume.Name) + log.Ctx(ctx).Info().Msgf("Delete PersistentVolume %s in namespace %s", existingPv.Name, namespace) + err = deploy.deleteCsiAzurePersistentVolumes(ctx, existingPv.Name) if err != nil { return nil, false, err } } log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) - pv := &corev1.PersistentVolume{} - err := populateCsiAzurePersistentVolume(pv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, nil) + newPv := &corev1.PersistentVolume{} + err := populateCsiAzurePersistentVolume(newPv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, identity) if err != nil { return nil, false, err } - desiredPersistentVolume, err := deploy.kubeclient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + desiredPersistentVolume, err := deploy.kubeclient.CoreV1().PersistentVolumes().Create(ctx, newPv, metav1.CreateOptions{}) return desiredPersistentVolume, true, err } diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 95278ffb1..591ca2489 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -960,7 +960,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // action deployComponent := deployment.radixDeployment.Spec.Components[0] - err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), desiredDeployment, deployComponent) + err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), desiredDeployment, &deployComponent) assert.Nil(t, err) existingPvcs, existingScs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment) From 98dde7c29f1f5bebae4f2537a389c92c21d2c201 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 26 Nov 2024 15:18:37 +0100 Subject: [PATCH 07/68] Added blobfuse properties --- pkg/apis/deployment/deployment.go | 2 +- pkg/apis/deployment/volumemount.go | 15 ++++++++++----- pkg/apis/deployment/volumemount_test.go | 22 +++++++++++----------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index e7803bae5..895234d68 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -480,7 +480,7 @@ func getLabelSelectorForBlobVolumeMountSecret(component v1.RadixCommonDeployComp } func getLabelSelectorForCsiAzureVolumeMountSecret(component v1.RadixCommonDeployComponent) string { - return fmt.Sprintf("%s=%s, %s in (%s, %s, %s, %s)", kube.RadixComponentLabel, component.GetName(), kube.RadixMountTypeLabel, string(v1.MountTypeBlobFuse2FuseCsiAzure), string(v1.MountTypeBlobFuse2Fuse2CsiAzure)) + return fmt.Sprintf("%s=%s, %s in (%s, %s)", kube.RadixComponentLabel, component.GetName(), kube.RadixMountTypeLabel, string(v1.MountTypeBlobFuse2FuseCsiAzure), string(v1.MountTypeBlobFuse2Fuse2CsiAzure)) } func (deploy *Deployment) maintainHistoryLimit(ctx context.Context, deploymentHistoryLimit int) { diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index 0cb498ec2..30558701e 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -518,10 +518,6 @@ func getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentStorage(compon } func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appName, namespace, componentName, pvcName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { - requestsVolumeMountSize, err := resource.ParseQuantity(getRadixBlobFuse2VolumeMountRequestsStorage(radixVolumeMount)) - if err != nil { - requestsVolumeMountSize = resource.MustParse("1Mi") - } pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, @@ -536,7 +532,7 @@ func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appNa Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{getVolumeMountAccessMode(radixVolumeMount)}, Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceStorage: requestsVolumeMountSize}, // it seems correct number is not needed for CSI driver + Requests: corev1.ResourceList{corev1.ResourceStorage: getVolumeCapacity(radixVolumeMount)}, }, VolumeName: pvName, }, @@ -544,6 +540,14 @@ func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appNa return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) } +func getVolumeCapacity(radixVolumeMount *radixv1.RadixVolumeMount) resource.Quantity { + requestsVolumeMountSize, err := resource.ParseQuantity(getRadixBlobFuse2VolumeMountRequestsStorage(radixVolumeMount)) + if err != nil { + return resource.MustParse("1Mi") + } + return requestsVolumeMountSize +} + func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, appName, volumeRootMount, namespace, componentName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount, identity *radixv1.Identity) error { identityClientId := getIdentityClientId(identity) useAzureIdentity := getUseAzureIdentity(identity, radixVolumeMount.UseAzureIdentity) @@ -556,6 +560,7 @@ func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, return err } persistentVolume.Spec.MountOptions = mountOptions + persistentVolume.Spec.Capacity = corev1.ResourceList{corev1.ResourceStorage: getVolumeCapacity(radixVolumeMount)} persistentVolume.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{getVolumeMountAccessMode(radixVolumeMount)} persistentVolume.Spec.ClaimRef = &corev1.ObjectReference{ APIVersion: "v1", diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 591ca2489..c074fb878 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -1382,7 +1382,7 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co if len(idOption) > 0 { mountOptions = append(mountOptions, idOption) } - reclaimPolicy := corev1.PersistentVolumeReclaimRetain + // reclaimPolicy := corev1.PersistentVolumeReclaimRetain sc := corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: props.persistentVolumeName, @@ -1395,15 +1395,15 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co }, }, Spec: corev1.PersistentVolumeSpec{ - Provisioner: props.scProvisioner, - Attr: map[string]string{ - csiPersistentVolumeProvisionerSecretNameParameter: props.scSecretName, - csiPersistentVolumeProvisionerSecretNamespaceParameter: props.namespace, - csiPersistentVolumeNodeStageSecretNameParameter: props.scSecretName, - csiPersistentVolumeNodeStageSecretNamespaceParameter: props.namespace, - }, - MountOptions: mountOptions, - ReclaimPolicy: &reclaimPolicy, + // Provisioner: props.scProvisioner, + // Attr: map[string]string{ + // csiPersistentVolumeProvisionerSecretNameParameter: props.scSecretName, + // csiPersistentVolumeProvisionerSecretNamespaceParameter: props.namespace, + // csiPersistentVolumeNodeStageSecretNameParameter: props.scSecretName, + // csiPersistentVolumeNodeStageSecretNamespaceParameter: props.namespace, + // }, + MountOptions: mountOptions, + // ReclaimPolicy: &reclaimPolicy, }, } setVolumeMountAttribute(props.radixVolumeMountType, props.radixStorageName, &sc) @@ -1452,7 +1452,7 @@ func createExpectedPvc(props expectedPvcScProperties, modify func(*corev1.Persis Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, // it seems correct number is not needed for CSI driver }, - PersistentVolumeName: utils.StringPtr(props.persistentVolumeName), + VolumeName: props.persistentVolumeName, }, } if modify != nil { From 9df11d0d08bd13c736b31c35559e943c1cd29ae3 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Wed, 27 Nov 2024 17:12:29 +0100 Subject: [PATCH 08/68] Added blobfuse properties --- pkg/apis/deployment/volumemount.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index 30558701e..255d89b0e 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -7,6 +7,7 @@ import ( "strings" commonUtils "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" @@ -534,7 +535,8 @@ func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appNa Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceStorage: getVolumeCapacity(radixVolumeMount)}, }, - VolumeName: pvName, + VolumeName: pvName, + StorageClassName: pointers.Ptr(""), // avoid to use the "default" storage class }, } return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) @@ -559,6 +561,7 @@ func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, if err != nil { return err } + persistentVolume.Spec.StorageClassName = "" persistentVolume.Spec.MountOptions = mountOptions persistentVolume.Spec.Capacity = corev1.ResourceList{corev1.ResourceStorage: getVolumeCapacity(radixVolumeMount)} persistentVolume.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{getVolumeMountAccessMode(radixVolumeMount)} From e6acdf3b0d034816bb269ba6ab004d3f84566de5 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 2 Dec 2024 17:15:37 +0100 Subject: [PATCH 09/68] Removed support for deprecated volume mount blob. Changing logic to create pvc and pv --- .../steps/internal/deployment_test.go | 2 +- pkg/apis/batch/kubejob.go | 2 +- pkg/apis/defaults/k8s/k8s.go | 11 +- pkg/apis/deployment/deployment.go | 4 - pkg/apis/deployment/deployment_test.go | 20 +- pkg/apis/deployment/kubedeployment.go | 3 +- pkg/apis/deployment/secrets.go | 32 +- pkg/apis/deployment/volumemount.go | 397 ++++++++---------- pkg/apis/deployment/volumemount_test.go | 169 ++++---- pkg/apis/internal/persistentvolume.go | 92 ++++ pkg/apis/internal/persistentvolumeclaim.go | 20 + pkg/apis/radix/v1/radixapptypes.go | 2 - pkg/apis/radixvalidators/validate_ra.go | 16 +- pkg/apis/radixvalidators/validate_ra_test.go | 33 -- pkg/apis/utils/persistentvolume.go | 89 ---- pkg/apis/utils/persistentvolumeclaim.go | 92 ---- 16 files changed, 414 insertions(+), 570 deletions(-) create mode 100644 pkg/apis/internal/persistentvolume.go create mode 100644 pkg/apis/internal/persistentvolumeclaim.go delete mode 100644 pkg/apis/utils/persistentvolume.go delete mode 100644 pkg/apis/utils/persistentvolumeclaim.go diff --git a/pipeline-runner/steps/internal/deployment_test.go b/pipeline-runner/steps/internal/deployment_test.go index cedc82162..bf07767c1 100644 --- a/pipeline-runner/steps/internal/deployment_test.go +++ b/pipeline-runner/steps/internal/deployment_test.go @@ -50,7 +50,7 @@ func TestConstructForTargetEnvironment_PicksTheCorrectEnvironmentConfig(t *testi }). WithVolumeMounts([]radixv1.RadixVolumeMount{ { - Type: radixv1.MountTypeBlob, + Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Container: "some-container", Path: "some-path", }, diff --git a/pkg/apis/batch/kubejob.go b/pkg/apis/batch/kubejob.go index 4f3e791b6..b3ab02674 100644 --- a/pkg/apis/batch/kubejob.go +++ b/pkg/apis/batch/kubejob.go @@ -174,7 +174,7 @@ func (s *syncer) getJobPodImagePullSecrets(rd *radixv1.RadixDeployment) []corev1 } func (s *syncer) getVolumes(ctx context.Context, namespace, environment string, batchJob *radixv1.RadixBatchJob, radixJobComponent *radixv1.RadixDeployJobComponent, radixDeploymentName string) ([]corev1.Volume, error) { - volumes, err := deployment.GetVolumes(ctx, s.kubeClient, s.kubeUtil, namespace, environment, radixJobComponent, radixDeploymentName) + volumes, err := deployment.GetVolumes(ctx, s.kubeClient, s.kubeUtil, namespace, environment, radixJobComponent, radixDeploymentName, nil) if err != nil { return nil, err } diff --git a/pkg/apis/defaults/k8s/k8s.go b/pkg/apis/defaults/k8s/k8s.go index 72a3bab71..7c69f9822 100644 --- a/pkg/apis/defaults/k8s/k8s.go +++ b/pkg/apis/defaults/k8s/k8s.go @@ -1,9 +1,10 @@ package k8s const ( - KindClusterRole = "ClusterRole" - KindClusterRoleBinding = "ClusterRoleBinding" - KindRole = "Role" - KindRoleBinding = "RoleBinding" - KindIngress = "Ingress" + KindClusterRole = "ClusterRole" + KindClusterRoleBinding = "ClusterRoleBinding" + KindRole = "Role" + KindRoleBinding = "RoleBinding" + KindIngress = "Ingress" + KindPersistentVolumeClaim = "PersistentVolumeClaim" ) diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 895234d68..b9f617d19 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -475,10 +475,6 @@ func getLabelSelectorForComponent(component v1.RadixCommonDeployComponent) strin return fmt.Sprintf("%s=%s", kube.RadixComponentLabel, component.GetName()) } -func getLabelSelectorForBlobVolumeMountSecret(component v1.RadixCommonDeployComponent) string { - return fmt.Sprintf("%s=%s, %s=%s", kube.RadixComponentLabel, component.GetName(), kube.RadixMountTypeLabel, string(v1.MountTypeBlob)) -} - func getLabelSelectorForCsiAzureVolumeMountSecret(component v1.RadixCommonDeployComponent) string { return fmt.Sprintf("%s=%s, %s in (%s, %s)", kube.RadixComponentLabel, component.GetName(), kube.RadixMountTypeLabel, string(v1.MountTypeBlobFuse2FuseCsiAzure), string(v1.MountTypeBlobFuse2Fuse2CsiAzure)) } diff --git a/pkg/apis/deployment/deployment_test.go b/pkg/apis/deployment/deployment_test.go index e1ad68b1d..daaea7631 100644 --- a/pkg/apis/deployment/deployment_test.go +++ b/pkg/apis/deployment/deployment_test.go @@ -211,12 +211,6 @@ func TestObjectSynced_MultiComponent_ContainsAllElements(t *testing.T) { WithPort("http", 3000). WithPublicPort("http"). WithVolumeMounts( - radixv1.RadixVolumeMount{ - Type: radixv1.MountTypeBlob, - Name: blobVolumeName, - Container: "some-container", - Path: "some-path", - }, radixv1.RadixVolumeMount{ Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: blobCsiAzureVolumeName, @@ -587,12 +581,6 @@ func TestObjectSynced_MultiJob_ContainsAllElements(t *testing.T) { "cpu": "501m", }). WithVolumeMounts( - radixv1.RadixVolumeMount{ - Type: radixv1.MountTypeBlob, - Name: blobVolumeName, - Container: "some-container", - Path: "some-path", - }, radixv1.RadixVolumeMount{ Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: blobCsiAzureVolumeName, @@ -3858,7 +3846,6 @@ func Test_ComponentSynced_VolumeAndMounts(t *testing.T) { utils.NewDeployComponentBuilder(). WithName(compName). WithVolumeMounts( - radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlob, Name: "blob", Container: "blobcontainer", Path: "blobpath"}, radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blobcsi", Storage: "blobcsistorage", Path: "blobcsipath"}, ), ), @@ -3868,8 +3855,8 @@ func Test_ComponentSynced_VolumeAndMounts(t *testing.T) { envNamespace := utils.GetEnvironmentNamespace(appName, environment) deployment, _ := client.AppsV1().Deployments(envNamespace).Get(context.Background(), compName, metav1.GetOptions{}) require.NotNil(t, deployment) - assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2, "incorrect number of volumes") - assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 2, "incorrect number of volumemounts") + assert.Len(t, deployment.Spec.Template.Spec.Volumes, 1, "incorrect number of volumes") + assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 1, "incorrect number of volumemounts") } func Test_JobSynced_VolumeAndMounts(t *testing.T) { @@ -3890,7 +3877,6 @@ func Test_JobSynced_VolumeAndMounts(t *testing.T) { utils.NewDeployJobComponentBuilder(). WithName(jobName). WithVolumeMounts( - radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlob, Name: "blob", Container: "blobcontainer", Path: "blobpath"}, radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blobcsi", Storage: "blobcsistorage", Path: "blobcsipath"}, ), ), @@ -3901,7 +3887,7 @@ func Test_JobSynced_VolumeAndMounts(t *testing.T) { deploymentList, _ := client.AppsV1().Deployments(envNamespace).List(context.Background(), metav1.ListOptions{LabelSelector: radixlabels.ForJobAuxObject(jobName, kube.RadixJobTypeManagerAux).String()}) require.Len(t, deploymentList.Items, 1) deployment := deploymentList.Items[0] - assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2, "incorrect number of volumes") + assert.Len(t, deployment.Spec.Template.Spec.Volumes, 1, "incorrect number of volumes") assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 0, "incorrect number of volumemounts") } diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index e29015279..11a6d1ad4 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -133,6 +133,7 @@ func (deploy *Deployment) getDesiredCreatedDeploymentConfig(ctx context.Context, err := deploy.setDesiredDeploymentProperties(ctx, deployComponent, desiredDeployment) return desiredDeployment, err } + func (deploy *Deployment) createJobAuxDeployment(deployComponent v1.RadixCommonDeployComponent) *appsv1.Deployment { jobName := deployComponent.GetName() jobAuxDeploymentName := getJobAuxObjectName(jobName) @@ -293,7 +294,7 @@ func (deploy *Deployment) setDesiredDeploymentProperties(ctx context.Context, de desiredDeployment.Spec.Template.Spec.Affinity = utils.GetAffinityForDeployComponent(ctx, deployComponent, appName, componentName) desiredDeployment.Spec.Template.Spec.Tolerations = utils.GetDeploymentPodSpecTolerations(deployComponent.GetNode()) - volumes, err := deploy.GetVolumesForComponent(ctx, deployComponent) + volumes, err := GetVolumes(ctx, deploy.kubeclient, deploy.kubeutil, deploy.getNamespace(), deploy.radixDeployment.Spec.Environment, deployComponent, deploy.radixDeployment.GetName(), desiredDeployment.Spec.Template.Spec.Volumes) if err != nil { return err } diff --git a/pkg/apis/deployment/secrets.go b/pkg/apis/deployment/secrets.go index cba9a881b..761df9d7e 100644 --- a/pkg/apis/deployment/secrets.go +++ b/pkg/apis/deployment/secrets.go @@ -116,25 +116,11 @@ func (deploy *Deployment) createOrUpdateSecretsForComponent(ctx context.Context, func (deploy *Deployment) createOrUpdateVolumeMountSecrets(ctx context.Context, namespace, componentName string, volumeMounts []radixv1.RadixVolumeMount) ([]string, error) { var volumeMountSecretsToManage []string for _, volumeMount := range volumeMounts { - switch GetCsiAzureVolumeMountType(&volumeMount) { - case radixv1.MountTypeBlob: - { - secretName, accountKey, accountName := deploy.getBlobFuseCredsSecrets(ctx, namespace, componentName, volumeMount.Name) - volumeMountSecretsToManage = append(volumeMountSecretsToManage, secretName) - err := deploy.createOrUpdateVolumeMountsSecrets(ctx, namespace, componentName, secretName, accountName, accountKey) - if err != nil { - return nil, err - } - } - case radixv1.MountTypeBlobFuse2FuseCsiAzure, radixv1.MountTypeBlobFuse2Fuse2CsiAzure: - { - secretName, accountKey, accountName := deploy.getCsiAzureVolumeMountCredsSecrets(ctx, namespace, componentName, volumeMount.Name) - volumeMountSecretsToManage = append(volumeMountSecretsToManage, secretName) - err := deploy.createOrUpdateCsiAzureVolumeMountsSecrets(ctx, namespace, componentName, &volumeMount, secretName, accountName, accountKey) - if err != nil { - return nil, err - } - } + secretName, accountKey, accountName := deploy.getCsiAzureVolumeMountCredsSecrets(ctx, namespace, componentName, volumeMount.Name) + volumeMountSecretsToManage = append(volumeMountSecretsToManage, secretName) + err := deploy.createOrUpdateCsiAzureVolumeMountsSecrets(ctx, namespace, componentName, &volumeMount, secretName, accountName, accountKey) + if err != nil { + return nil, err } } return volumeMountSecretsToManage, nil @@ -228,18 +214,12 @@ func (deploy *Deployment) listSecretsForComponent(ctx context.Context, component } func (deploy *Deployment) listSecretsForVolumeMounts(ctx context.Context, component radixv1.RadixCommonDeployComponent) ([]*v1.Secret, error) { - blobVolumeMountSecret := getLabelSelectorForBlobVolumeMountSecret(component) - secrets, err := deploy.listSecrets(ctx, blobVolumeMountSecret) - if err != nil { - return nil, err - } csiAzureVolumeMountSecret := getLabelSelectorForCsiAzureVolumeMountSecret(component) csiSecrets, err := deploy.listSecrets(ctx, csiAzureVolumeMountSecret) if err != nil { return nil, err } - secrets = append(secrets, csiSecrets...) - return secrets, err + return csiSecrets, nil } func (deploy *Deployment) listSecrets(ctx context.Context, labelSelector string) ([]*v1.Secret, error) { diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index 255d89b0e..bc979a290 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -8,10 +8,13 @@ import ( commonUtils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/pointers" + "github.com/equinor/radix-common/utils/slice" "github.com/equinor/radix-operator/pkg/apis/defaults" + "github.com/equinor/radix-operator/pkg/apis/defaults/k8s" + "github.com/equinor/radix-operator/pkg/apis/internal" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" - "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/google/uuid" "github.com/rs/zerolog/log" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -22,41 +25,45 @@ import ( ) const ( - persistentVolumeClaimKind = "PersistentVolumeClaim" - blobfuseDriver = "azure/blobfuse" defaultMountOptions = "--file-cache-timeout-in-seconds=120" blobFuseVolumeNameTemplate = "blobfuse-%s-%s" // blobfuse-- blobFuseVolumeNodeMountPathTemplate = "/tmp/%s/%s/%s/%s/%s/%s" // /tmp////// - csiVolumeNameTemplate = "%s-%s-%s-%s" // --- - csiPersistentVolumeClaimNameTemplate = "pvc-%s-%s" // pvc-- - csiPersistentVolumeNameTemplate = "sc-%s-%s" // sc-- - csiVolumeNodeMountPathTemplate = "%s/%s/%s/%s/%s/%s" // ///// - - csiPersistentVolumeProvisionerSecretNameParameter = "csi.storage.k8s.io/provisioner-secret-name" // Secret name, containing storage account name and key - csiPersistentVolumeProvisionerSecretNamespaceParameter = "csi.storage.k8s.io/provisioner-secret-namespace" // namespace of the secret - csiPersistentVolumeNodeStageSecretNameParameter = "csi.storage.k8s.io/node-stage-secret-name" // Usually equal to csiPersistentVolumeProvisionerSecretNameParameter - csiPersistentVolumeNodeStageSecretNamespaceParameter = "csi.storage.k8s.io/node-stage-secret-namespace" // Usually equal to csiPersistentVolumeProvisionerSecretNamespaceParameter - csiVolumeMountAttributeContainerName = "containerName" // Container name - foc container storages - csiPersistentVolumeTmpPathMountOption = "tmp-path" // Path within the node, where the volume mount has been mounted to - csiPersistentVolumeGidMountOption = "gid" // Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting - csiPersistentVolumeUidMountOption = "uid" // Volume mount owner UserID. Used instead of GroupID - csiPersistentVolumeUseAdlsMountOption = "use-adls" // Use ADLS or Block Blob - csiPersistentVolumeStreamingEnabledMountOption = "streaming" // Enable Streaming - csiPersistentVolumeStreamingCacheMountOption = "stream-cache-mb" // Limit total amount of data being cached in memory to conserve memory - csiPersistentVolumeStreamingMaxBlocksPerFileMountOption = "max-blocks-per-file" // Maximum number of blocks to be cached in memory for streaming - csiPersistentVolumeStreamingMaxBuffersMountOption = "max-buffers" // The total number of buffers to be cached in memory (in MB). - csiPersistentVolumeStreamingBlockSizeMountOption = "block-size-mb" // The size of each block to be cached in memory (in MB). - csiPersistentVolumeStreamingBufferSizeMountOption = "buffer-size-mb" // The size of each buffer to be cached in memory (in MB). - csiVolumeMountAttributeProtocol = "protocol" // Protocol - csiPersistentVolumeProtocolParameterFuse = "fuse" // Protocol "blobfuse" - csiPersistentVolumeProtocolParameterFuse2 = "fuse2" // Protocol "blobfuse2" - csiVolumeMountAttributeStorageAccount = "storageAccount" - csiVolumeMountAttributeClientID = "clientID" - csiVolumeMountAttributeResourceGroup = "resourcegroup" - azureCsiDriverVolumeHandle = "azure-csi-driver-volume-handle" + csiVolumeNameTemplate = "%s-%s-%s-%s" // --- + csiPersistentVolumeClaimNameTemplate = "pvc-%s-%s" // pvc-- + csiPersistentVolumeNameTemplate = "pv-radixvolumemount-%s" // pv- + csiVolumeNodeMountPathTemplate = "%s/%s/%s/%s/%s/%s" // ///// + + csiPersistentVolumeProvisionerSecretNameParameter = "csi.storage.k8s.io/provisioner-secret-name" // Secret name, containing storage account name and key + csiPersistentVolumeProvisionerSecretNamespaceParameter = "csi.storage.k8s.io/provisioner-secret-namespace" // namespace of the secret + csiPersistentVolumeNodeStageSecretNameParameter = "csi.storage.k8s.io/node-stage-secret-name" // Usually equal to csiPersistentVolumeProvisionerSecretNameParameter + csiPersistentVolumeNodeStageSecretNamespaceParameter = "csi.storage.k8s.io/node-stage-secret-namespace" // Usually equal to csiPersistentVolumeProvisionerSecretNamespaceParameter + csiVolumeMountAttributeContainerName = "containerName" // Container name - foc container storages + csiPersistentVolumeTmpPathMountOption = "tmp-path" // Path within the node, where the volume mount has been mounted to + csiPersistentVolumeGidMountOption = "gid" // Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting + csiPersistentVolumeUidMountOption = "uid" // Volume mount owner UserID. Used instead of GroupID + csiPersistentVolumeUseAdlsMountOption = "use-adls" // Use ADLS or Block Blob + csiPersistentVolumeStreamingEnabledMountOption = "streaming" // Enable Streaming + csiPersistentVolumeStreamingCacheMountOption = "stream-cache-mb" // Limit total amount of data being cached in memory to conserve memory + csiPersistentVolumeStreamingMaxBlocksPerFileMountOption = "max-blocks-per-file" // Maximum number of blocks to be cached in memory for streaming + csiPersistentVolumeStreamingMaxBuffersMountOption = "max-buffers" // The total number of buffers to be cached in memory (in MB). + csiPersistentVolumeStreamingBlockSizeMountOption = "block-size-mb" // The size of each block to be cached in memory (in MB). + csiPersistentVolumeStreamingBufferSizeMountOption = "buffer-size-mb" // The size of each buffer to be cached in memory (in MB). + csiVolumeMountAttributeProtocol = "protocol" // Protocol + csiPersistentVolumeProtocolParameterFuse = "fuse" // Protocol "blobfuse" + csiPersistentVolumeProtocolParameterFuse2 = "fuse2" // Protocol "blobfuse2" + csiVolumeMountAttributeStorageAccount = "storageAccount" + csiVolumeMountAttributeClientID = "clientID" + csiVolumeMountAttributeResourceGroup = "resourcegroup" + csiVolumeMountAttributeSecretNamespace = "secretnamespace" + csiVolumeMountAttributePvName = "csi.storage.k8s.io/pv/name" + csiVolumeMountAttributePvcName = "csi.storage.k8s.io/pv/name" + csiVolumeMountAttributePvcNamespace = "csi.storage.k8s.io/pv/name" + csiVolumeMountAnnotationProvisionedBy = "pv.kubernetes.io/provisioned-by" + csiVolumeMountAnnotationProvisionerDeletionSecretName = "volume.kubernetes.io/provisioner-deletion-secret-name" + csiVolumeMountAnnotationProvisionerDeletionSecretNamespace = "volume.kubernetes.io/provisioner-deletion-secret-namespace" csiSecretStoreDriver = "secrets-store.csi.k8s.io" csiVolumeSourceVolumeAttrSecretProviderClassName = "secretProviderClass" @@ -72,7 +79,8 @@ const ( ) var ( - csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}} + csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}} + functionalPersistentVolumePhases = []corev1.PersistentVolumePhase{corev1.VolumePending, corev1.VolumeBound, corev1.VolumeAvailable} ) // isKnownCsiAzureVolumeMount Supported volume mount type CSI Azure Blob volume @@ -144,12 +152,9 @@ func getCsiAzureKeyVaultSecretMountPath(azureKeyVault radixv1.RadixAzureKeyVault return *azureKeyVault.Path } -func getBlobFuseVolumeMountName(volumeMount *radixv1.RadixVolumeMount, componentName string) string { - return trimVolumeNameToValidLength(fmt.Sprintf(blobFuseVolumeNameTemplate, componentName, volumeMount.Name)) -} - -func getCsiAzureVolumeMountName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { - csiVolumeType, err := getCsiRadixVolumeTypeIdForName(volumeMount) +// volumeName: --- +func getCsiAzureVolumeMountName(componentName string, volumeMount *radixv1.RadixVolumeMount) (string, error) { + csiVolumeType, err := getCsiRadixVolumeTypeId(volumeMount) if err != nil { return "", err } @@ -179,7 +184,7 @@ func GetCsiAzureVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) radi } } -func getCsiRadixVolumeTypeIdForName(radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { +func getCsiRadixVolumeTypeId(radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { if radixVolumeMount.BlobFuse2 != nil { switch radixVolumeMount.BlobFuse2.Protocol { case radixv1.BlobFuse2ProtocolFuse2, "": @@ -195,16 +200,10 @@ func getCsiRadixVolumeTypeIdForName(radixVolumeMount *radixv1.RadixVolumeMount) return "", fmt.Errorf("unknown volume mount type %s", radixVolumeMount.Type) } -// GetVolumesForComponent Gets volumes for Radix deploy component or job -func (deploy *Deployment) GetVolumesForComponent(ctx context.Context, deployComponent radixv1.RadixCommonDeployComponent) ([]corev1.Volume, error) { - return GetVolumes(ctx, deploy.kubeclient, deploy.kubeutil, deploy.getNamespace(), deploy.radixDeployment.Spec.Environment, deployComponent, deploy.radixDeployment.GetName()) -} - // GetVolumes Get volumes of a component by RadixVolumeMounts -func GetVolumes(ctx context.Context, kubeclient kubernetes.Interface, kubeutil *kube.Kube, namespace string, environment string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.Volume, error) { +func GetVolumes(ctx context.Context, kubeclient kubernetes.Interface, kubeutil *kube.Kube, namespace string, environment string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { var volumes []corev1.Volume - - volumeMountVolumes, err := getComponentVolumeMountVolumes(ctx, kubeclient, namespace, environment, deployComponent) + volumeMountVolumes, err := getComponentVolumeMountVolumes(ctx, kubeclient, namespace, environment, deployComponent, existingVolumes) if err != nil { return nil, err } @@ -248,7 +247,7 @@ func getComponentSecretRefsAzureKeyVaultVolumes(ctx context.Context, kubeutil *k case "azure": volume.VolumeSource.CSI = &corev1.CSIVolumeSource{ Driver: csiSecretStoreDriver, - ReadOnly: commonUtils.BoolPtr(true), + ReadOnly: pointers.Ptr(true), VolumeAttributes: map[string]string{csiVolumeSourceVolumeAttrSecretProviderClassName: secretProviderClass.Name}, } @@ -271,27 +270,20 @@ func getComponentSecretRefsAzureKeyVaultVolumes(ctx context.Context, kubeutil *k return volumes, nil } -func getComponentVolumeMountVolumes(ctx context.Context, kubeclient kubernetes.Interface, namespace string, environment string, deployComponent radixv1.RadixCommonDeployComponent) ([]corev1.Volume, error) { +func getComponentVolumeMountVolumes(ctx context.Context, kubeClient kubernetes.Interface, namespace, environment string, deployComponent radixv1.RadixCommonDeployComponent, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { + componentName := deployComponent.GetName() + existingVolumesMap := getVolumesMap(existingVolumes) var volumes []corev1.Volume - - volumeSourceFunc := func(volumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { - switch { - case volumeMount.HasDeprecatedVolume(): - return getComponentVolumeMountDeprecatedVolumeSource(ctx, volumeMount, namespace, environment, deployComponent.GetName(), kubeclient) - case volumeMount.HasBlobFuse2(): - return getComponentVolumeMountBlobFuse2VolumeSource(ctx, volumeMount, namespace, deployComponent.GetName(), kubeclient) - case volumeMount.HasEmptyDir(): - return getComponentVolumeMountEmptyDirVolumeSource(volumeMount.EmptyDir), nil - } - return nil, fmt.Errorf("missing configuration for volumeMount %s", volumeMount.Name) - } - for _, volumeMount := range deployComponent.GetVolumeMounts() { - volumeSource, err := volumeSourceFunc(&volumeMount) + volumeName, err := getVolumeMountVolumeName(&volumeMount, componentName) if err != nil { return nil, err } - volumeName, err := getVolumeMountVolumeName(&volumeMount, deployComponent.GetName()) + var existingVolumeSource *corev1.VolumeSource + if existingVolume, ok := existingVolumesMap[volumeName]; ok { + existingVolumeSource = &existingVolume.VolumeSource + } + volumeSource, err := getVolumeSource(ctx, kubeClient, namespace, environment, componentName, &volumeMount, existingVolumeSource) if err != nil { return nil, err } @@ -303,21 +295,36 @@ func getComponentVolumeMountVolumes(ctx context.Context, kubeclient kubernetes.I return volumes, nil } -func getComponentVolumeMountDeprecatedVolumeSource(ctx context.Context, volumeMount *radixv1.RadixVolumeMount, namespace, environment, componentName string, kubeclient kubernetes.Interface) (*corev1.VolumeSource, error) { +func getVolumesMap(volumes []corev1.Volume) map[string]corev1.Volume { + return slice.Reduce(volumes, make(map[string]corev1.Volume), func(acc map[string]corev1.Volume, volume corev1.Volume) map[string]corev1.Volume { + if volume.PersistentVolumeClaim != nil { + acc[volume.Name] = volume + } + return acc + }) +} + +func getVolumeSource(ctx context.Context, kubeClient kubernetes.Interface, namespace, environment, componentName string, volumeMount *radixv1.RadixVolumeMount, existingVolumeSource *corev1.VolumeSource) (*corev1.VolumeSource, error) { + switch { + case volumeMount.HasDeprecatedVolume(): + return getComponentVolumeMountDeprecatedVolumeSource(ctx, kubeClient, namespace, environment, componentName, volumeMount, existingVolumeSource) + case volumeMount.HasBlobFuse2(): + return getCsiAzureVolumeSource(ctx, kubeClient, namespace, componentName, volumeMount, existingVolumeSource) + case volumeMount.HasEmptyDir(): + return getComponentVolumeMountEmptyDirVolumeSource(volumeMount.EmptyDir), nil + } + return nil, fmt.Errorf("missing configuration for volumeMount %s", volumeMount.Name) +} + +func getComponentVolumeMountDeprecatedVolumeSource(ctx context.Context, kubeClient kubernetes.Interface, namespace, environment, componentName string, volumeMount *radixv1.RadixVolumeMount, existingVolumeSource *corev1.VolumeSource) (*corev1.VolumeSource, error) { switch volumeMount.Type { - case radixv1.MountTypeBlob: - return getBlobFuseVolume(namespace, environment, componentName, volumeMount), nil case radixv1.MountTypeBlobFuse2FuseCsiAzure: - return getCsiAzureVolume(ctx, kubeclient, namespace, componentName, volumeMount) + return getCsiAzureVolumeSource(ctx, kubeClient, namespace, componentName, volumeMount, existingVolumeSource) } return nil, fmt.Errorf("unsupported volume type %s", volumeMount.Type) } -func getComponentVolumeMountBlobFuse2VolumeSource(ctx context.Context, volumeMount *radixv1.RadixVolumeMount, namespace, componentName string, kubeclient kubernetes.Interface) (*corev1.VolumeSource, error) { - return getCsiAzureVolume(ctx, kubeclient, namespace, componentName, volumeMount) -} - func getComponentVolumeMountEmptyDirVolumeSource(spec *radixv1.RadixEmptyDirVolumeMount) *corev1.VolumeSource { return &corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{ @@ -326,15 +333,19 @@ func getComponentVolumeMountEmptyDirVolumeSource(spec *radixv1.RadixEmptyDirVolu } } -func getCsiAzureVolume(ctx context.Context, kubeclient kubernetes.Interface, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { - existingNotTerminatingPvcForComponentStorage, err := getPvcNotTerminating(ctx, kubeclient, namespace, componentName, radixVolumeMount) +func getCsiAzureVolumeSource(ctx context.Context, kubeClient kubernetes.Interface, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, existingVolumeSource *corev1.VolumeSource) (*corev1.VolumeSource, error) { + existingNotTerminatingPvcList, err := getPvcNotTerminating(ctx, kubeClient, namespace, componentName, radixVolumeMount) + if err != nil { + return nil, err + } + pvList, err := getFunctionalCsiPersistentVolumesForNamespace(ctx, kubeClient, namespace) if err != nil { return nil, err } var pvcName string - if existingNotTerminatingPvcForComponentStorage != nil { - pvcName = existingNotTerminatingPvcForComponentStorage.Name + if existingNotTerminatingPvcList != nil { + pvcName = existingNotTerminatingPvcList.Name } else { pvcName, err = createCsiAzurePersistentVolumeClaimName(componentName, radixVolumeMount) if err != nil { @@ -353,38 +364,32 @@ func getVolumeMountVolumeName(volumeMount *radixv1.RadixVolumeMount, componentNa case volumeMount.HasDeprecatedVolume(): return getVolumeMountDeprecatedVolumeName(volumeMount, componentName) case volumeMount.HasBlobFuse2(): - return getVolumeMountBlobFuse2VolumeName(volumeMount, componentName) + return getCsiAzureVolumeMountName(componentName, volumeMount) } return fmt.Sprintf("radix-vm-%s", volumeMount.Name), nil } -func getVolumeMountBlobFuse2VolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { - return getCsiAzureVolumeMountName(volumeMount, componentName) -} - func getVolumeMountDeprecatedVolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { switch volumeMount.Type { - case radixv1.MountTypeBlob: - return getBlobFuseVolumeMountName(volumeMount, componentName), nil case radixv1.MountTypeBlobFuse2FuseCsiAzure: - return getCsiAzureVolumeMountName(volumeMount, componentName) + return getCsiAzureVolumeMountName(componentName, volumeMount) } - return "", fmt.Errorf("unsupported type %s", volumeMount.Type) } func getPvcNotTerminating(ctx context.Context, kubeclient kubernetes.Interface, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { - existingPvcForComponentStorage, err := kubeclient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentStorage(componentName, radixVolumeMount.Name), + pvcList, err := kubeclient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentVolumeMount(componentName, radixVolumeMount.Name), }) if err != nil { return nil, err } - existingPvcs := sortPvcsByCreatedTimestampDesc(existingPvcForComponentStorage.Items) + existingPvcs := sortPvcsByCreatedTimestampDesc(pvcList.Items) if len(existingPvcs) == 0 { return nil, nil } + for _, pvc := range existingPvcs { switch pvc.Status.Phase { case corev1.ClaimPending, corev1.ClaimBound: @@ -395,64 +400,17 @@ func getPvcNotTerminating(ctx context.Context, kubeclient kubernetes.Interface, } func createCsiAzurePersistentVolumeClaimName(componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { - volumeName, err := getCsiAzureVolumeMountName(radixVolumeMount, componentName) + volumeName, err := getCsiAzureVolumeMountName(componentName, radixVolumeMount) if err != nil { return "", err } - return fmt.Sprintf(csiPersistentVolumeClaimNameTemplate, volumeName, strings.ToLower(commonUtils.RandString(5))), nil // volumeName: --- + return fmt.Sprintf(csiPersistentVolumeClaimNameTemplate, volumeName, strings.ToLower(commonUtils.RandString(5))), nil } -func getCsiAzurePersistentVolumeName(namespace, volumeName string) string { - return fmt.Sprintf(csiPersistentVolumeNameTemplate, namespace, volumeName) // volumeName: --- -} - -func getBlobFuseVolume(namespace, environment, componentName string, volumeMount *radixv1.RadixVolumeMount) *corev1.VolumeSource { - secretName := defaults.GetBlobFuseCredsSecretName(componentName, volumeMount.Name) - - flexVolumeOptions := make(map[string]string) - flexVolumeOptions["name"] = volumeMount.Name - flexVolumeOptions["container"] = volumeMount.Container - flexVolumeOptions["mountoptions"] = defaultMountOptions - flexVolumeOptions["tmppath"] = fmt.Sprintf(blobFuseVolumeNodeMountPathTemplate, namespace, componentName, environment, radixv1.MountTypeBlob, volumeMount.Name, volumeMount.Container) - - return &corev1.VolumeSource{ - FlexVolume: &corev1.FlexVolumeSource{ - Driver: blobfuseDriver, - Options: flexVolumeOptions, - SecretRef: &corev1.LocalObjectReference{ - Name: secretName, - }, - }, - } +func getCsiAzurePersistentVolumeName() string { + return fmt.Sprintf(csiPersistentVolumeNameTemplate, uuid.New().String()) } -func (deploy *Deployment) createOrUpdateVolumeMountsSecrets(ctx context.Context, namespace, componentName, secretName string, accountName, accountKey []byte) error { - blobfusecredsSecret := corev1.Secret{ - Type: blobfuseDriver, - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Labels: map[string]string{ - kube.RadixAppLabel: deploy.registration.Name, - kube.RadixComponentLabel: componentName, - kube.RadixMountTypeLabel: string(radixv1.MountTypeBlob), - }, - }, - } - - // Will need to set fake data in order to apply the secret. The user then need to set data to real values - data := make(map[string][]byte) - data[defaults.BlobFuseCredsAccountKeyPart] = accountKey - data[defaults.BlobFuseCredsAccountNamePart] = accountName - - blobfusecredsSecret.Data = data - - _, err := deploy.kubeutil.ApplySecret(ctx, namespace, &blobfusecredsSecret) //nolint:staticcheck // must be updated to use UpdateSecret or CreateSecret - if err != nil { - return err - } - - return nil -} func (deploy *Deployment) createOrUpdateCsiAzureVolumeMountsSecrets(ctx context.Context, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, accountName, accountKey []byte) error { secret := corev1.Secret{ Type: corev1.SecretTypeOpaque, @@ -490,10 +448,26 @@ func (deploy *Deployment) garbageCollectVolumeMountsSecretsNoLongerInSpecForComp return deploy.GarbageCollectSecrets(ctx, secrets, excludeSecretNames) } -func (deploy *Deployment) getCsiAzurePersistentVolume(ctx context.Context, namespace, componentName string) (*corev1.PersistentVolumeList, error) { - return deploy.kubeclient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{ - LabelSelector: getLabelSelectorForCsiAzurePersistentVolume(namespace, componentName), - }) +func getFunctionalCsiPersistentVolumesForNamespace(ctx context.Context, kubeClient kubernetes.Interface, namespace string) ([]corev1.PersistentVolume, error) { + pvList, err := kubeClient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + return slice.FindAll(pvList.Items, func(pv corev1.PersistentVolume) bool { + return pvIsForCsiDriver(pv) && pvIsForNamespace(pv, namespace) && pvIsFunctional(pv) + }), nil +} + +func pvIsForCsiDriver(pv corev1.PersistentVolume) bool { + return pv.Spec.CSI != nil +} + +func pvIsForNamespace(pv corev1.PersistentVolume, namespace string) bool { + return pv.Spec.ClaimRef != nil && pv.Spec.ClaimRef.Namespace == namespace +} + +func pvIsFunctional(pv corev1.PersistentVolume) bool { + return slice.Any(functionalPersistentVolumePhases, func(phase corev1.PersistentVolumePhase) bool { return pv.Status.Phase == phase }) } func (deploy *Deployment) getCsiAzurePersistentVolumeClaims(ctx context.Context, namespace, componentName string) (*corev1.PersistentVolumeClaimList, error) { @@ -506,16 +480,12 @@ func (deploy *Deployment) getPersistentVolumesForPvc(ctx context.Context) (*core return deploy.kubeclient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) } -func getLabelSelectorForCsiAzurePersistentVolume(namespace, componentName string) string { - return fmt.Sprintf("%s=%s, %s=%s, %s in (%s, %s)", kube.RadixNamespace, namespace, kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure)) -} - func getLabelSelectorForCsiAzurePersistenceVolumeClaim(componentName string) string { return fmt.Sprintf("%s=%s, %s in (%s, %s)", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure)) } -func getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentStorage(componentName, radixVolumeMountName string) string { - return fmt.Sprintf("%s=%s, %s in (%s, %s), %s = %s", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure), kube.RadixVolumeMountNameLabel, radixVolumeMountName) +func getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentVolumeMount(componentName, radixVolumeMountName string) string { + return fmt.Sprintf("%s=%s, %s=%s", kube.RadixComponentLabel, componentName, kube.RadixVolumeMountNameLabel, radixVolumeMountName) } func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appName, namespace, componentName, pvcName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { @@ -567,13 +537,13 @@ func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, persistentVolume.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{getVolumeMountAccessMode(radixVolumeMount)} persistentVolume.Spec.ClaimRef = &corev1.ObjectReference{ APIVersion: "v1", - Kind: persistentVolumeClaimKind, + Kind: k8s.KindPersistentVolumeClaim, Namespace: namespace, Name: pvName, } persistentVolume.Spec.CSI = &corev1.CSIPersistentVolumeSource{ Driver: provisionerBlobCsiAzure, - VolumeHandle: azureCsiDriverVolumeHandle, + VolumeHandle: getVolumeHandle(namespace, componentName, pvName, radixVolumeMount), VolumeAttributes: getCsiAzurePersistentVolumeAttributes(radixVolumeMount, pvName, namespace, useAzureIdentity, identityClientId), } if !useAzureIdentity { @@ -583,6 +553,12 @@ func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, return nil } +// Specify a value the driver can use to uniquely identify the share in the cluster. +// https://github.com/kubernetes-csi/csi-driver-smb/blob/master/docs/driver-parameters.md#pvpvc-usage +func getVolumeHandle(namespace, componentName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount) string { + return fmt.Sprintf("%s#%s#%s#%s", namespace, componentName, pvName, getRadixVolumeMountStorage(radixVolumeMount)) +} + func getUseAzureIdentity(identity *radixv1.Identity, useAzureIdentity *bool) bool { return len(getIdentityClientId(identity)) > 0 && useAzureIdentity != nil && *useAzureIdentity } @@ -596,11 +572,11 @@ func getIdentityClientId(identity *radixv1.Identity) string { func getCsiAzurePersistentVolumeAnnotations(csiVolumeCredSecretName, namespace string, useAzureIdentity bool) map[string]string { annotationsMap := map[string]string{ - "pv.kubernetes.io/provisioned-by": provisionerBlobCsiAzure, + csiVolumeMountAnnotationProvisionedBy: provisionerBlobCsiAzure, } if !useAzureIdentity { - annotationsMap["volume.kubernetes.io/provisioner-deletion-secret-name"] = csiVolumeCredSecretName - annotationsMap["volume.kubernetes.io/provisioner-deletion-secret-namespace"] = namespace + annotationsMap[csiVolumeMountAnnotationProvisionerDeletionSecretName] = csiVolumeCredSecretName + annotationsMap[csiVolumeMountAnnotationProvisionerDeletionSecretNamespace] = namespace } return annotationsMap } @@ -631,18 +607,19 @@ func getCsiAzurePersistentVolumeAttributes(radixVolumeMount *radixv1.RadixVolume attributes[csiVolumeMountAttributeResourceGroup] = radixVolumeMount.BlobFuse2.ResourceGroup } } - attributes["csi.storage.k8s.io/pv/name"] = pvName - attributes["csi.storage.k8s.io/pvc/name"] = pvName - attributes["csi.storage.k8s.io/pvc/namespace"] = namespace + attributes[csiVolumeMountAttributePvName] = pvName + attributes[csiVolumeMountAttributePvcName] = pvName + attributes[csiVolumeMountAttributePvcNamespace] = namespace if !useAzureIdentity { - attributes["secretnamespace"] = namespace + attributes[csiVolumeMountAttributeSecretNamespace] = namespace } - // ? storage.kubernetes.io/csiProvisionerIdentity: 1731647415428-2825-blob.csi.azure.com + // Do not specify the key storage.kubernetes.io/csiProvisionerIdentity in csi.volumeAttributes in PV specification. This key indicates dynamically provisioned PVs + // It looks like: storage.kubernetes.io/csiProvisionerIdentity: 1731647415428-2825-blob.csi.azure.com return attributes } func getCsiAzurePersistentVolumeMountOptions(volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) ([]string, error) { - csiVolumeTypeId, err := getCsiRadixVolumeTypeIdForName(radixVolumeMount) + csiVolumeTypeId, err := getCsiRadixVolumeTypeId(radixVolumeMount) if err != nil { return nil, err } @@ -766,33 +743,28 @@ func getRadixBlobFuse2VolumeMountBindingMode(radixVolumeMount *radixv1.RadixVolu } func (deploy *Deployment) deletePersistentVolumeClaim(ctx context.Context, namespace, pvcName string) error { - if len(namespace) > 0 && len(pvcName) > 0 { - return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}) + if len(namespace) == 0 || len(pvcName) == 0 { + log.Ctx(ctx).Debug().Msgf("Skip deleting PVC - namespace %s or name %s is empty", namespace, pvcName) + return nil } - log.Ctx(ctx).Debug().Msgf("Skip deleting PVC - namespace %s or name %s is empty", namespace, pvcName) - return nil -} - -func (deploy *Deployment) deleteCsiAzurePersistentVolumes(ctx context.Context, pvName string) error { - if len(pvName) > 0 { - return deploy.kubeclient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}) + if err := deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { + return err } - log.Ctx(ctx).Debug().Msg("Skip deleting PersistentVolume - name is empty") return nil } func (deploy *Deployment) deletePersistentVolume(ctx context.Context, pvName string) error { - if len(pvName) > 0 { - return deploy.kubeclient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}) + if len(pvName) == 0 { + log.Ctx(ctx).Debug().Msg("Skip deleting PersistentVolume - name is empty") + return nil + } + if err := deploy.kubeclient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { + return err } - log.Ctx(ctx).Debug().Msg("Skip deleting PersistentVolume - name is empty") return nil } func getRadixVolumeMountStorage(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.Type == radixv1.MountTypeBlob { - return radixVolumeMount.Container // Outdated - } blobFuse2VolumeMountContainer := getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) if len(blobFuse2VolumeMountContainer) != 0 { return blobFuse2VolumeMountContainer @@ -806,17 +778,17 @@ func (deploy *Deployment) garbageCollectOrphanedCsiAzurePersistentVolumes(ctx co return err } for _, pv := range pvList.Items { - if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.Kind != persistentVolumeClaimKind || - !knownCSIDriver(pv.Spec.CSI) || - pv.Status.Phase != corev1.VolumeReleased { + if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.Kind != k8s.KindPersistentVolumeClaim || !knownCSIDriver(pv.Spec.CSI) { + continue + } + if !(pv.Status.Phase == corev1.VolumeReleased || pv.Status.Phase == corev1.VolumeFailed) { continue } if _, ok := excludePvcNames[pv.Spec.ClaimRef.Name]; ok { continue } log.Ctx(ctx).Info().Msgf("Delete orphaned Csi Azure PersistantVolume %s of PersistantVolumeClaim %s", pv.Name, pv.Spec.ClaimRef.Name) - err := deploy.deletePersistentVolume(ctx, pv.Name) - if err != nil { + if err = deploy.deletePersistentVolume(ctx, pv.Name); err != nil { return err } } @@ -837,7 +809,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont appName := deploy.radixDeployment.Spec.AppName componentName := desiredDeployment.ObjectMeta.Name volumeRootMount := "/tmp" // TODO: add to environment variable, so this volume can be mounted to external disk - pvList, err := deploy.getCsiAzurePersistentVolume(ctx, namespace, componentName) + pvList, err := getFunctionalCsiPersistentVolumesForNamespace(ctx, deploy.kubeclient, namespace) if err != nil { return err } @@ -846,8 +818,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont return err } - pvMap := utils.GetPersistentVolumeMap(&pvList.Items) - pvcMap := utils.GetPersistentVolumeClaimMap(&pvcList.Items) + existingPvcMap := internal.GetPersistentVolumeClaimMap(&pvcList.Items, false) radixVolumeMountMap := deploy.getRadixVolumeMountMapByCsiAzureVolumeMountName(componentName) var actualPersistentVolumeNames []string actualPvcNames, err := deploy.getCurrentlyUsedPersistentVolumeClaims(ctx, namespace) @@ -863,15 +834,15 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont if !existsRadixVolumeMount { return fmt.Errorf("not found Radix volume mount for desired volume %s", volume.Name) } - pv, pvIsCreated, err := deploy.getOrCreateCsiAzurePersistentVolume(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, volume.Name, pvMap, identity) + pv, pvIsCreated, err := deploy.getOrCreateCsiAzurePersistentVolume(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, pvList, volume.PersistentVolumeClaim.ClaimName, identity) if err != nil { return err } - actualPersistentVolumeNames = append(actualPersistentVolumeNames, pv.Name) - pvc, err := deploy.createCsiAzurePersistentVolumeClaim(ctx, pv, pvIsCreated, appName, namespace, componentName, radixVolumeMount, volume.PersistentVolumeClaim.ClaimName, pvcMap) + pvc, err := deploy.createCsiAzurePersistentVolumeClaim(ctx, pv, pvIsCreated, appName, namespace, componentName, radixVolumeMount, volume.PersistentVolumeClaim.ClaimName, existingPvcMap) if err != nil { return err } + actualPersistentVolumeNames = append(actualPersistentVolumeNames, pv.Name) volume.PersistentVolumeClaim.ClaimName = pvc.Name actualPvcNames[pvc.Name] = struct{}{} } @@ -920,13 +891,11 @@ func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersist } pvName := pvc.Spec.VolumeName log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolumeClaim %s in namespace %s", pvc.Name, namespace) - err := deploy.deletePersistentVolumeClaim(ctx, namespace, pvc.Name) - if err != nil { + if err := deploy.deletePersistentVolumeClaim(ctx, namespace, pvc.Name); err != nil { return err } log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolume %s in namespace %s", pvName, namespace) - err = deploy.deletePersistentVolume(ctx, pvName) - if err != nil { + if err := deploy.deletePersistentVolume(ctx, pvName); err != nil { return err } } @@ -952,33 +921,37 @@ func (deploy *Deployment) createCsiAzurePersistentVolumeClaim(ctx context.Contex return deploy.createPersistentVolumeClaim(ctx, appName, namespace, componentName, persistentVolumeClaimName, pv.Name, radixVolumeMount) } -func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Context, appName, volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, volumeName string, pvMap map[string]*corev1.PersistentVolume, identity *radixv1.Identity) (*corev1.PersistentVolume, bool, error) { - pvName := getCsiAzurePersistentVolumeName(namespace, volumeName) - if existingPv, exists := pvMap[pvName]; exists { - desiredPv := existingPv.DeepCopy() - err := populateCsiAzurePersistentVolume(desiredPv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, identity) - if err != nil { - return nil, false, err - } - if equal, err := utils.EqualPersistentVolumes(existingPv, desiredPv); equal || err != nil { - return existingPv, false, err +func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Context, appName, volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, pvList []corev1.PersistentVolume, pvcName string, identity *radixv1.Identity) (*corev1.PersistentVolume, bool, error) { + + for _, existingPv := range pvList { + if existingPv.Spec.ClaimRef != nil && existingPv.Spec.ClaimRef.Name == pvcName { + desiredPv := existingPv.DeepCopy() + err := populateCsiAzurePersistentVolume(desiredPv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, identity) + if err != nil { + return nil, false, err + } + if equal, err := internal.EqualPersistentVolumes(&existingPv, desiredPv); equal || err != nil { + return &existingPv, false, err + } + + log.Ctx(ctx).Info().Msgf("Delete PersistentVolume %s in namespace %s", existingPv.Name, namespace) + if err = deploy.deletePersistentVolume(ctx, existingPv.Name); err != nil { + return nil, false, err + } + continue } - log.Ctx(ctx).Info().Msgf("Delete PersistentVolume %s in namespace %s", existingPv.Name, namespace) - err = deploy.deleteCsiAzurePersistentVolumes(ctx, existingPv.Name) + pvName := getCsiAzurePersistentVolumeName() + log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) + newPv := &corev1.PersistentVolume{} + err := populateCsiAzurePersistentVolume(newPv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, identity) if err != nil { return nil, false, err } + desiredPersistentVolume, err := deploy.kubeclient.CoreV1().PersistentVolumes().Create(ctx, newPv, metav1.CreateOptions{}) + return desiredPersistentVolume, true, err } - - log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) - newPv := &corev1.PersistentVolume{} - err := populateCsiAzurePersistentVolume(newPv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, identity) - if err != nil { - return nil, false, err - } - desiredPersistentVolume, err := deploy.kubeclient.CoreV1().PersistentVolumes().Create(ctx, newPv, metav1.CreateOptions{}) - return desiredPersistentVolume, true, err + return nil, false, fmt.Errorf("failed to create PersistentVolume %s in namespace %s", pvName, namespace) // TODO check if this is correct } func (deploy *Deployment) getRadixVolumeMountMapByCsiAzureVolumeMountName(componentName string) map[string]*radixv1.RadixVolumeMount { @@ -1005,7 +978,7 @@ func findCsiAzureVolumeForComponent(volumeMountMap map[string]*radixv1.RadixVolu continue } radixVolumeMount := radixVolumeMount - volumeMountName, err := getCsiAzureVolumeMountName(&radixVolumeMount, componentName) + volumeMountName, err := getCsiAzureVolumeMountName(componentName, &radixVolumeMount) if err != nil { return false } diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index c074fb878..7b33a8715 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -2,12 +2,15 @@ package deployment import ( "context" + "encoding/json" "fmt" "strings" "testing" + commonUtils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/config" + "github.com/equinor/radix-operator/pkg/apis/internal" "github.com/equinor/radix-operator/pkg/apis/kube" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" @@ -24,6 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes" kubefake "k8s.io/client-go/kubernetes/fake" secretProviderClient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" @@ -216,47 +220,6 @@ func (suite *VolumeMountTestSuite) Test_FailBlobCsiAzureVolumeMounts() { }) } -// Blobfuse support has been deprecated, this test to be deleted, when Blobfuse logic is deleted -func (suite *VolumeMountTestSuite) Test_BlobfuseAzureVolumeMounts() { - scenarios := []volumeMountTestScenario{ - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlob, Name: "volume1", Container: "storageName1", Path: "TestPath1"}, - expectedVolumeName: "blobfuse-app-volume1", - }, - { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlob, Name: "volume2", Container: "storageName2", Path: "TestPath2"}, - expectedVolumeName: "blobfuse-app-volume2", - }, - } - suite.T().Run("One Blobfuse Azure volume mount", func(t *testing.T) { - t.Parallel() - component := utils.NewDeployComponentBuilder().WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(&component, "") - assert.Nil(t, err) - assert.Equal(t, 1, len(volumeMounts)) - mount := volumeMounts[0] - assert.Equal(t, scenarios[0].expectedVolumeName, mount.Name) - assert.Equal(t, scenarios[0].radixVolumeMount.Path, mount.MountPath) - }) - suite.T().Run("Multiple Blobfuse Azure volume mount", func(t *testing.T) { - t.Parallel() - component := utils.NewDeployComponentBuilder().WithName("app"). - WithVolumeMounts(scenarios[0].radixVolumeMount, scenarios[1].radixVolumeMount). - BuildComponent() - - volumeMounts, err := GetRadixDeployComponentVolumeMounts(&component, "") - assert.Nil(t, err) - for idx, testCase := range scenarios { - assert.Equal(t, 2, len(volumeMounts)) - assert.Equal(t, testCase.expectedVolumeName, volumeMounts[idx].Name) - assert.Equal(t, testCase.radixVolumeMount.Path, volumeMounts[idx].MountPath) - } - }) -} - func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { namespace := "some-namespace" environment := "some-env" @@ -265,7 +228,7 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { t.Parallel() testEnv := getTestEnv() component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts().BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") + volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) assert.Nil(t, err) assert.Len(t, volumes, 0) }) @@ -284,18 +247,13 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-volume-with-long-name-storageN-", }, } - blobFuseScenario := volumeMountTestScenario{ - name: "Blob Azure FlexVolume", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlob, Name: "volume1", Container: "storage1", Path: "path1"}, - expectedVolumeName: "blobfuse-some-component-volume1", - } suite.T().Run("CSI Azure volumes", func(t *testing.T) { t.Parallel() testEnv := getTestEnv() for _, scenario := range scenarios { t.Logf("Scenario %s", scenario.name) component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") + volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) assert.Nil(t, err) assert.Len(t, volumes, 1) volume := volumes[0] @@ -309,29 +267,12 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { assert.Contains(t, volume.PersistentVolumeClaim.ClaimName, scenario.expectedPvcNamePrefix) } }) - suite.T().Run("Blobfuse-flex volume", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(blobFuseScenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") - assert.Nil(t, err) - assert.Len(t, volumes, 1) - volume := volumes[0] - assert.Equal(t, blobFuseScenario.expectedVolumeName, volume.Name) - assert.Nil(t, volume.PersistentVolumeClaim) - assert.NotNil(t, volume.FlexVolume) - assert.Equal(t, "azure/blobfuse", volume.FlexVolume.Driver) - assert.Equal(t, "volume1", volume.FlexVolume.Options["name"]) - assert.Equal(t, "storage1", volume.FlexVolume.Options["container"]) - assert.Equal(t, "--file-cache-timeout-in-seconds=120", volume.FlexVolume.Options["mountoptions"]) - assert.Equal(t, "/tmp/some-namespace/some-component/some-env/blob/volume1/storage1", volume.FlexVolume.Options["tmppath"]) - }) suite.T().Run("CSI Azure and Blobfuse-flex volumes", func(t *testing.T) { t.Parallel() testEnv := getTestEnv() for _, scenario := range append(scenarios, blobFuseScenario) { component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") + volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) assert.Nil(t, err) assert.Len(t, volumes, 1) volume := volumes[0] @@ -351,7 +292,7 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { {Type: "unsupported-type", Name: "volume1", Container: "storage1", Path: "path1"}, } component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(mounts...).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") + volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) assert.Len(t, volumes, 0) assert.NotNil(t, err) assert.Equal(t, "unsupported volume type unsupported-type", err.Error()) @@ -399,7 +340,7 @@ func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { _, _ = testEnv.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), &scenario.pvc, metav1.CreateOptions{}) component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") + volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) assert.Nil(t, err) assert.Len(t, volumes, 1) assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name) @@ -415,7 +356,7 @@ func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { t.Logf("Scenario %s for volume mount type %s, PVC status phase '%v'", scenario.name, string(GetCsiAzureVolumeMountType(&scenario.radixVolumeMount)), scenario.pvc.Status.Phase) component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "") + volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) assert.Nil(t, err) assert.Len(t, volumes, 1) assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name) @@ -462,7 +403,7 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{}) deployComponent := deployment.radixDeployment.Spec.Components[0] - volumes, err := deployment.GetVolumesForComponent(context.Background(), &deployComponent) + volumes, err := GetVolumes(context.Background(), deployment.kubeclient, deployment.kubeutil, deployment.getNamespace(), deployment.radixDeployment.Spec.Environment, &deployComponent, deployment.radixDeployment.GetName(), nil) assert.Nil(t, err) assert.Len(t, volumes, 0) @@ -479,7 +420,7 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{scenario.radixVolumeMount}) deployComponent := deployment.radixDeployment.Spec.Components[0] - volumes, err := deployment.GetVolumesForComponent(context.Background(), &deployComponent) + volumes, err := GetVolumes(context.Background(), deployment.kubeclient, deployment.kubeutil, deployment.getNamespace(), deployment.radixDeployment.Spec.Environment, &deployComponent, deployment.radixDeployment.GetName(), nil) assert.Nil(t, err) assert.Len(t, volumes, 1) @@ -965,10 +906,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPvcs, existingScs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment) assert.Nil(t, err) - equalPvcLists, err := utils.EqualPvcLists(&scenario.existingPvcsAfterTestRun, &existingPvcs, true) + equalPvcLists, err := equalPvcLists(&scenario.existingPvcsAfterTestRun, &existingPvcs, true) assert.Nil(t, err) assert.True(t, equalPvcLists) - equalPersistentVolumeLists, err := utils.EqualPersistentVolumeLists(&scenario.existingPersistentVolumeAfterTestRun, &existingScs) + equalPersistentVolumeLists, err := equalPersistentVolumeLists(&scenario.existingPersistentVolumeAfterTestRun, &existingScs) assert.Nil(t, err) assert.True(t, equalPersistentVolumeLists) } @@ -1097,7 +1038,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureKeyVaultResources( t.Logf("created secret provider class %s", spc.Name) } } - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, radixDeployComponent, deployment.radixDeployment.GetName()) + volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, radixDeployComponent, deployment.radixDeployment.GetName(), nil) assert.Nil(t, err) assert.Len(t, volumes, len(scenario.expectedVolumeProps)) if len(scenario.expectedVolumeProps) == 0 { @@ -1501,3 +1442,83 @@ func createBlobFuse2RadixVolumeMount(props expectedPvcScProperties, modify func( } return volumeMount } + +func equalPersistentVolumeLists(list1, list2 *[]corev1.PersistentVolume) (bool, error) { + if len(*list1) != len(*list2) { + return false, fmt.Errorf("different PersistentVolume list sizes: %v, %v", len(*list1), len(*list2)) + } + map1 := internal.GetPersistentVolumeMap(list1) + map2 := internal.GetPersistentVolumeMap(list2) + for pvName, pv1 := range map1 { + pv2, ok := map2[pvName] + if !ok { + return false, fmt.Errorf("PersistentVolume not found by name %s in second list", pvName) + } + if equal, err := internal.EqualPersistentVolumes(pv1, pv2); err != nil || !equal { + return false, err + } + } + return true, nil +} + +func equalPvcLists(pvcList1, pvcList2 *[]corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (bool, error) { + if len(*pvcList1) != len(*pvcList2) { + return false, nil + } + pvcMap1 := internal.GetPersistentVolumeClaimMap(pvcList1, ignoreRandomPostfixInName) + pvcMap2 := internal.GetPersistentVolumeClaimMap(pvcList2, ignoreRandomPostfixInName) + for pvcName, pvc1 := range pvcMap1 { + pvc2, ok := pvcMap2[pvcName] + if !ok { + return false, fmt.Errorf("PVS not found by name %s in second list", pvcName) + } + if equal, err := equalPvcs(pvc1, pvc2, ignoreRandomPostfixInName); err != nil || !equal { + return false, err + } + } + return true, nil +} + +func equalPvcs(pvc1 *corev1.PersistentVolumeClaim, pvc2 *corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (bool, error) { + pvc1Copy, labels1 := getPvcCopyWithLabels(pvc1, ignoreRandomPostfixInName) + pvc2Copy, labels2 := getPvcCopyWithLabels(pvc2, ignoreRandomPostfixInName) + patchBytes, err := getPvcPatch(pvc1Copy, pvc2Copy) + if err != nil { + return false, err + } + if !utils.EqualStringMaps(labels1, labels2) { + return false, fmt.Errorf("PVC-s labels are not equal") + } + if !kube.IsEmptyPatch(patchBytes) { + return false, fmt.Errorf("PVC-s are not equal: %s", patchBytes) + } + return true, nil +} + +func getPvcCopyWithLabels(pvc *corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (*corev1.PersistentVolumeClaim, map[string]string) { + pvcCopy := pvc.DeepCopy() + pvcCopy.ObjectMeta.ManagedFields = nil // HACK: to avoid ManagedFields comparison + if ignoreRandomPostfixInName { + pvcCopy.ObjectMeta.Name = commonUtils.ShortenString(pvcCopy.ObjectMeta.Name, 6) + } + // to avoid label order variations + labels := pvcCopy.ObjectMeta.Labels + pvcCopy.ObjectMeta.Labels = map[string]string{} + return pvcCopy, labels +} + +func getPvcPatch(pvc1, pvc2 *corev1.PersistentVolumeClaim) ([]byte, error) { + json1, err := json.Marshal(pvc1) + if err != nil { + return nil, err + } + json2, err := json.Marshal(pvc2) + if err != nil { + return nil, err + } + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(json1, json2, corev1.PersistentVolumeClaim{}) + if err != nil { + return nil, err + } + return patchBytes, nil +} diff --git a/pkg/apis/internal/persistentvolume.go b/pkg/apis/internal/persistentvolume.go new file mode 100644 index 000000000..851e07fed --- /dev/null +++ b/pkg/apis/internal/persistentvolume.go @@ -0,0 +1,92 @@ +package internal + +import ( + "strings" + + "github.com/equinor/radix-common/utils/slice" + "github.com/equinor/radix-operator/pkg/apis/utils" + corev1 "k8s.io/api/core/v1" +) + +// GetPersistentVolumeMap Get map from PersistentVolumeList with name as key +func GetPersistentVolumeMap(pvList *[]corev1.PersistentVolume) map[string]*corev1.PersistentVolume { + pvMap := make(map[string]*corev1.PersistentVolume) + for _, pv := range *pvList { + pv := pv + pvMap[pv.Name] = &pv + } + return pvMap +} + +// EqualPersistentVolumes Compare two PersistentVolumes +func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) (bool, error) { + // Ignore for now, due to during transition period this would affect existing volume mounts, managed by a provisioner + // if !utils.EqualStringMaps(pv1.GetLabels(), pv2.GetLabels()) { + // return false, nil + // } + if !utils.EqualStringMaps(getAnnotations(pv1), getAnnotations(pv2)) { + return false, nil + } + if !utils.EqualStringMaps(pv1.Spec.CSI.VolumeAttributes, pv2.Spec.CSI.VolumeAttributes) { + return false, nil + } + if !utils.EqualStringMaps(getMountOptionsMap(pv1.Spec.MountOptions), getMountOptionsMap(pv2.Spec.MountOptions)) { + return false, nil + } + if pv1.Spec.StorageClassName != pv2.Spec.StorageClassName || + pv1.Spec.Capacity[corev1.ResourceStorage] != pv2.Spec.Capacity[corev1.ResourceStorage] || + len(pv1.Spec.AccessModes) != len(pv2.Spec.AccessModes) || + (len(pv1.Spec.AccessModes) != 1 && pv1.Spec.AccessModes[0] != pv2.Spec.AccessModes[0]) || + pv1.Spec.CSI.Driver != pv2.Spec.CSI.Driver { + return false, nil + } + if pv1.Spec.CSI.NodeStageSecretRef != nil { + if pv2.Spec.CSI.NodeStageSecretRef == nil || pv1.Spec.CSI.NodeStageSecretRef.Name != pv2.Spec.CSI.NodeStageSecretRef.Name { + return false, nil + } + } else if pv2.Spec.CSI.NodeStageSecretRef != nil { + return false, nil + } + if pv1.Spec.ClaimRef != nil { + if pv2.Spec.ClaimRef == nil || pv1.Spec.ClaimRef.Name != pv2.Spec.ClaimRef.Name { + return false, nil + } + } else if pv2.Spec.ClaimRef != nil { + return false, nil + } + return true, nil +} + +func getAnnotations(pv *corev1.PersistentVolume) map[string]string { + annotations := make(map[string]string) + for key, value := range pv.GetAnnotations() { + if key == "kubectl.kubernetes.io/last-applied-configuration" { + continue // ignore automatically added annotation(s) + } + annotations[key] = value + } + return annotations +} + +func getMountOptionsMap(mountOptions []string) map[string]string { + return slice.Reduce(mountOptions, make(map[string]string), func(acc map[string]string, item string) map[string]string { + if len(item) == 0 { + return acc + } + itemParts := strings.Split(item, "=") + key, value := "", "" + if len(itemParts) > 0 { + key = itemParts[0] + } + if key == "--tmp-path" { + return acc // ignore tmp-path, which eventually can be introduced + } + if len(itemParts) > 1 { + value = itemParts[1] + } + if len(key) > 0 { + acc[key] = value + } + return acc + }) +} diff --git a/pkg/apis/internal/persistentvolumeclaim.go b/pkg/apis/internal/persistentvolumeclaim.go new file mode 100644 index 000000000..ee105e738 --- /dev/null +++ b/pkg/apis/internal/persistentvolumeclaim.go @@ -0,0 +1,20 @@ +package internal + +import ( + "github.com/equinor/radix-operator/pkg/apis/utils" + corev1 "k8s.io/api/core/v1" +) + +// GetPersistentVolumeClaimMap Get map from PersistentVolumeClaim with name as key +func GetPersistentVolumeClaimMap(pvcList *[]corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) map[string]*corev1.PersistentVolumeClaim { + pvcMap := make(map[string]*corev1.PersistentVolumeClaim) + for _, pvc := range *pvcList { + pvc := pvc + name := pvc.Name + if ignoreRandomPostfixInName { + name = utils.ShortenString(name, 6) + } + pvcMap[name] = &pvc + } + return pvcMap +} diff --git a/pkg/apis/radix/v1/radixapptypes.go b/pkg/apis/radix/v1/radixapptypes.go index 77b5128a0..850375e62 100644 --- a/pkg/apis/radix/v1/radixapptypes.go +++ b/pkg/apis/radix/v1/radixapptypes.go @@ -1058,8 +1058,6 @@ type MountType string // These are valid types of mount const ( - // MountTypeBlob Use of azure/blobfuse flexvolume - MountTypeBlob MountType = "blob" // MountTypeBlobFuse2FuseCsiAzure Use of azure/csi driver for blobfuse2, protocol Fuse in Azure storage account MountTypeBlobFuse2FuseCsiAzure MountType = "azure-blob" // MountTypeBlobFuse2Fuse2CsiAzure Use of azure/csi driver for blobfuse2, protocol Fuse2 in Azure storage account diff --git a/pkg/apis/radixvalidators/validate_ra.go b/pkg/apis/radixvalidators/validate_ra.go index 0367e075e..3c61b6845 100644 --- a/pkg/apis/radixvalidators/validate_ra.go +++ b/pkg/apis/radixvalidators/validate_ra.go @@ -1504,27 +1504,17 @@ func validateVolumeMounts(volumeMounts []radixv1.RadixVolumeMount) error { } func validateVolumeMountDeprecatedSource(v *radixv1.RadixVolumeMount) error { - if !slices.Contains([]radixv1.MountType{radixv1.MountTypeBlob, radixv1.MountTypeBlobFuse2FuseCsiAzure}, v.Type) { + if v.Type != radixv1.MountTypeBlobFuse2FuseCsiAzure { return volumeMountDeprecatedSourceValidationError(ErrVolumeMountInvalidType) } - if len(v.RequestsStorage) > 0 { if _, err := resource.ParseQuantity(v.RequestsStorage); err != nil { return volumeMountDeprecatedSourceValidationError(fmt.Errorf("%w. %w", ErrVolumeMountInvalidRequestsStorage, err)) } } - - switch v.Type { - case radixv1.MountTypeBlob: - if len(v.Container) == 0 { - return volumeMountDeprecatedSourceValidationError(ErrVolumeMountMissingContainer) - } - case radixv1.MountTypeBlobFuse2FuseCsiAzure: - if len(v.Storage) == 0 { - return volumeMountDeprecatedSourceValidationError(ErrVolumeMountMissingStorage) - } + if v.Type == radixv1.MountTypeBlobFuse2FuseCsiAzure && len(v.Storage) == 0 { + return volumeMountDeprecatedSourceValidationError(ErrVolumeMountMissingStorage) } - return nil } diff --git a/pkg/apis/radixvalidators/validate_ra_test.go b/pkg/apis/radixvalidators/validate_ra_test.go index 8a1bd3876..4a03c647e 100644 --- a/pkg/apis/radixvalidators/validate_ra_test.go +++ b/pkg/apis/radixvalidators/validate_ra_test.go @@ -1281,39 +1281,6 @@ func Test_ValidationOfVolumeMounts_Errors(t *testing.T) { updateRA: setComponentAndJobsVolumeMounts, expectedError: radixvalidators.ErrVolumeMountMissingType, }, - "multiple types: deprecated source and blobfuse2": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Name: "anyname", - Path: "/path", - Type: radixv1.MountTypeBlob, - BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{}, - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountMultipleTypes, - }, - "multiple types: blobfuse2 and emptyDir": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Name: "anyname", - Path: "/path", - Type: radixv1.MountTypeBlob, - BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{}, - EmptyDir: &radixv1.RadixEmptyDirVolumeMount{}, - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountMultipleTypes, - }, "deprecated blob: valid": { volumeMounts: func() []radixv1.RadixVolumeMount { volumeMounts := []radixv1.RadixVolumeMount{ diff --git a/pkg/apis/utils/persistentvolume.go b/pkg/apis/utils/persistentvolume.go deleted file mode 100644 index 61283f660..000000000 --- a/pkg/apis/utils/persistentvolume.go +++ /dev/null @@ -1,89 +0,0 @@ -package utils - -import ( - "encoding/json" - "fmt" - - "github.com/equinor/radix-operator/pkg/apis/kube" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/strategicpatch" -) - -// GetPersistentVolumeMap Get map from PersistentVolumeList with name as key -func GetPersistentVolumeMap(pvList *[]corev1.PersistentVolume) map[string]*corev1.PersistentVolume { - pvMap := make(map[string]*corev1.PersistentVolume) - for _, pv := range *pvList { - pv := pv - pvMap[pv.Name] = &pv - } - return pvMap -} - -// EqualPersistentVolumeLists Compare two PersistentVolume lists -func EqualPersistentVolumeLists(list1, list2 *[]corev1.PersistentVolume) (bool, error) { - if len(*list1) != len(*list2) { - return false, fmt.Errorf("different PersistentVolume list sizes: %v, %v", len(*list1), len(*list2)) - } - map1 := GetPersistentVolumeMap(list1) - map2 := GetPersistentVolumeMap(list2) - for pvName, pv1 := range map1 { - pv2, ok := map2[pvName] - if !ok { - return false, fmt.Errorf("PersistentVolume not found by name %s in second list", pvName) - } - if equal, err := EqualPersistentVolumes(pv1, pv2); err != nil || !equal { - return false, err - } - } - return true, nil -} - -// EqualPersistentVolumes Compare two PersistentVolumes -func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) (bool, error) { - pv1Copy, labels1, attribs1, mountOptions1 := getPersistentVolumeCopyWithCollections(pv1) - pv2Copy, labels2, attribs2, mountOptions2 := getPersistentVolumeCopyWithCollections(pv2) - patchBytes, err := getPersistentVolumePatch(pv1Copy, pv2Copy) - if err != nil { - return false, err - } - if !EqualStringMaps(labels1, labels2) { - return false, nil // PersistentVolume labels are not equal - } - if !EqualStringMaps(attribs1, attribs2) { - return false, nil // PersistentVolume parameters are not equal - } - if !EqualStringLists(mountOptions1, mountOptions2) { - return false, nil // PersistentVolume-es MountOptions are not equal - } - if !kube.IsEmptyPatch(patchBytes) { - return false, nil // PersistentVolume properties are not equal - } - return true, nil -} - -func getPersistentVolumeCopyWithCollections(pv *corev1.PersistentVolume) (*corev1.PersistentVolume, map[string]string, map[string]string, []string) { - pvCopy := pv.DeepCopy() - pvCopy.ObjectMeta.ManagedFields = nil // HACK: to avoid ManagedFields comparison - // to avoid label order variations - labels := pvCopy.ObjectMeta.Labels - pvCopy.ObjectMeta.Labels = map[string]string{} - // to avoid Attribs order variations - pvAttribs := pvCopy.Spec.CSI.VolumeAttributes - pvCopy.Spec.CSI.VolumeAttributes = map[string]string{} - // to avoid MountOptions order variations - mountOptions := pvCopy.Spec.MountOptions - pvCopy.Spec.MountOptions = []string{} - return pvCopy, labels, pvAttribs, mountOptions -} - -func getPersistentVolumePatch(pv1, pv2 *corev1.PersistentVolume) ([]byte, error) { - json1, err := json.Marshal(pv1) - if err != nil { - return []byte{}, err - } - json2, err := json.Marshal(pv2) - if err != nil { - return []byte{}, err - } - return strategicpatch.CreateTwoWayMergePatch(json1, json2, corev1.PersistentVolume{}) -} diff --git a/pkg/apis/utils/persistentvolumeclaim.go b/pkg/apis/utils/persistentvolumeclaim.go deleted file mode 100644 index 1880037ce..000000000 --- a/pkg/apis/utils/persistentvolumeclaim.go +++ /dev/null @@ -1,92 +0,0 @@ -package utils - -import ( - "encoding/json" - "fmt" - - "github.com/equinor/radix-operator/pkg/apis/kube" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/strategicpatch" -) - -// GetPersistentVolumeClaimMap Get map from PersistentVolumeClaim with name as key -func GetPersistentVolumeClaimMap(pvcList *[]corev1.PersistentVolumeClaim) map[string]*corev1.PersistentVolumeClaim { - return getPersistentVolumeClaimMap(pvcList, false) -} - -func getPersistentVolumeClaimMap(pvcList *[]corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) map[string]*corev1.PersistentVolumeClaim { - pvcMap := make(map[string]*corev1.PersistentVolumeClaim) - for _, pvc := range *pvcList { - pvc := pvc - name := pvc.Name - if ignoreRandomPostfixInName { - name = ShortenString(name, 6) - } - pvcMap[name] = &pvc - } - return pvcMap -} - -// EqualPvcLists Compare two PersistentVolumeClaim lists. When ignoreRandomPostfixInName=true - last 6 chars of the name (e.g.'-abc12') are ignored during comparison -func EqualPvcLists(pvcList1, pvcList2 *[]corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (bool, error) { - if len(*pvcList1) != len(*pvcList2) { - return false, nil - } - pvcMap1 := getPersistentVolumeClaimMap(pvcList1, ignoreRandomPostfixInName) - pvcMap2 := getPersistentVolumeClaimMap(pvcList2, ignoreRandomPostfixInName) - for pvcName, pvc1 := range pvcMap1 { - pvc2, ok := pvcMap2[pvcName] - if !ok { - return false, fmt.Errorf("PVS not found by name %s in second list", pvcName) - } - if equal, err := EqualPvcs(pvc1, pvc2, ignoreRandomPostfixInName); err != nil || !equal { - return false, err - } - } - return true, nil -} - -// EqualPvcs Compare two PersistentVolumeClaim pointers -func EqualPvcs(pvc1 *corev1.PersistentVolumeClaim, pvc2 *corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (bool, error) { - pvc1Copy, labels1 := getPvcCopyWithLabels(pvc1, ignoreRandomPostfixInName) - pvc2Copy, labels2 := getPvcCopyWithLabels(pvc2, ignoreRandomPostfixInName) - patchBytes, err := getPvcPatch(pvc1Copy, pvc2Copy) - if err != nil { - return false, err - } - if !EqualStringMaps(labels1, labels2) { - return false, fmt.Errorf("PVC-s labels are not equal") - } - if !kube.IsEmptyPatch(patchBytes) { - return false, fmt.Errorf("PVC-s are not equal: %s", patchBytes) - } - return true, nil -} - -func getPvcCopyWithLabels(pvc *corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (*corev1.PersistentVolumeClaim, map[string]string) { - pvcCopy := pvc.DeepCopy() - pvcCopy.ObjectMeta.ManagedFields = nil // HACK: to avoid ManagedFields comparison - if ignoreRandomPostfixInName { - pvcCopy.ObjectMeta.Name = ShortenString(pvcCopy.ObjectMeta.Name, 6) - } - // to avoid label order variations - labels := pvcCopy.ObjectMeta.Labels - pvcCopy.ObjectMeta.Labels = map[string]string{} - return pvcCopy, labels -} - -func getPvcPatch(pvc1, pvc2 *corev1.PersistentVolumeClaim) ([]byte, error) { - json1, err := json.Marshal(pvc1) - if err != nil { - return nil, err - } - json2, err := json.Marshal(pvc2) - if err != nil { - return nil, err - } - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(json1, json2, corev1.PersistentVolumeClaim{}) - if err != nil { - return nil, err - } - return patchBytes, nil -} From f2545e035e17bc5e7aa61275154fd503b31ed5ee Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 3 Dec 2024 17:13:21 +0100 Subject: [PATCH 10/68] Changing logic to create pvc and pv --- pkg/apis/deployment/kubedeployment.go | 18 +- pkg/apis/deployment/volumemount.go | 391 ++++++++++----------- pkg/apis/deployment/volumemount_test.go | 22 +- pkg/apis/internal/persistentvolume.go | 28 +- pkg/apis/internal/persistentvolumeclaim.go | 21 ++ 5 files changed, 234 insertions(+), 246 deletions(-) diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index 11a6d1ad4..2635523bb 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -30,26 +30,19 @@ func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployCompone // If component has manual override or HorizontalScaling is nil then delete hpa if exists before updating deployment if deployComponent.GetReplicasOverride() != nil || deployComponent.GetHorizontalScaling() == nil { - err = deploy.deleteScaledObjectIfExists(ctx, deployComponent.GetName()) - if err != nil { + if err = deploy.deleteScaledObjectIfExists(ctx, deployComponent.GetName()); err != nil { return err } - - err = deploy.deleteTargetAuthenticationIfExists(ctx, deployComponent.GetName()) - if err != nil { + if err = deploy.deleteTargetAuthenticationIfExists(ctx, deployComponent.GetName()); err != nil { return err } } - - err = deploy.createOrUpdateCsiAzureVolumeResources(ctx, desiredDeployment, deployComponent) - if err != nil { + if err = deploy.createOrUpdateCsiAzureVolumeResources(ctx, deployComponent, desiredDeployment); err != nil { return err } - err = deploy.handleJobAuxDeployment(ctx, deployComponent, desiredDeployment) - if err != nil { + if err = deploy.handleJobAuxDeployment(ctx, deployComponent, desiredDeployment); err != nil { return err } - return deploy.kubeutil.ApplyDeployment(ctx, deploy.radixDeployment.Namespace, currentDeployment, desiredDeployment) } @@ -66,8 +59,7 @@ func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, deployComp selector := labels.Set(desiredJobAuxDeployment.Spec.Selector.MatchLabels).AsSelector() if currentJobAuxDeployment != nil && !selector.Matches(labels.Set(currentJobAuxDeployment.Spec.Template.Labels)) { log.Ctx(ctx).Info().Msgf("Deleting outdated deployment (label selector does not match) %s", currentJobAuxDeployment.GetName()) - err = deploy.kubeutil.DeleteDeployment(ctx, deploy.radixDeployment.Namespace, currentJobAuxDeployment.Name) - if err != nil { + if err = deploy.kubeutil.DeleteDeployment(ctx, deploy.radixDeployment.Namespace, currentJobAuxDeployment.Name); err != nil { return err } diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index bc979a290..e524da116 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -2,6 +2,7 @@ package deployment import ( "context" + "errors" "fmt" "sort" "strings" @@ -25,16 +26,11 @@ import ( ) const ( - blobfuseDriver = "azure/blobfuse" - defaultMountOptions = "--file-cache-timeout-in-seconds=120" - - blobFuseVolumeNameTemplate = "blobfuse-%s-%s" // blobfuse-- - blobFuseVolumeNodeMountPathTemplate = "/tmp/%s/%s/%s/%s/%s/%s" // /tmp////// - + csiVolumeTypeBlobFuse2ProtocolFuse = "csi-az-blob" + csiVolumeTypeBlobFuse2ProtocolFuse2 = "csi-blobfuse2-fuse2" csiVolumeNameTemplate = "%s-%s-%s-%s" // --- csiPersistentVolumeClaimNameTemplate = "pvc-%s-%s" // pvc-- csiPersistentVolumeNameTemplate = "pv-radixvolumemount-%s" // pv- - csiVolumeNodeMountPathTemplate = "%s/%s/%s/%s/%s/%s" // ///// csiPersistentVolumeProvisionerSecretNameParameter = "csi.storage.k8s.io/provisioner-secret-name" // Secret name, containing storage account name and key csiPersistentVolumeProvisionerSecretNamespaceParameter = "csi.storage.k8s.io/provisioner-secret-namespace" // namespace of the secret @@ -79,8 +75,9 @@ const ( ) var ( - csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}} - functionalPersistentVolumePhases = []corev1.PersistentVolumePhase{corev1.VolumePending, corev1.VolumeBound, corev1.VolumeAvailable} + csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}} + functionalPersistentVolumePhases = []corev1.PersistentVolumePhase{corev1.VolumePending, corev1.VolumeBound, corev1.VolumeAvailable} + functionalPersistentVolumeClaimPhases = []corev1.PersistentVolumeClaimPhase{corev1.ClaimPending, corev1.ClaimBound} ) // isKnownCsiAzureVolumeMount Supported volume mount type CSI Azure Blob volume @@ -188,14 +185,13 @@ func getCsiRadixVolumeTypeId(radixVolumeMount *radixv1.RadixVolumeMount) (string if radixVolumeMount.BlobFuse2 != nil { switch radixVolumeMount.BlobFuse2.Protocol { case radixv1.BlobFuse2ProtocolFuse2, "": - return "csi-blobfuse2-fuse2", nil + return csiVolumeTypeBlobFuse2ProtocolFuse2, nil default: return "", fmt.Errorf("unknown blobfuse2 protocol %s", radixVolumeMount.BlobFuse2.Protocol) } } - switch radixVolumeMount.Type { - case radixv1.MountTypeBlobFuse2FuseCsiAzure: - return "csi-az-blob", nil + if radixVolumeMount.Type == radixv1.MountTypeBlobFuse2FuseCsiAzure { + return csiVolumeTypeBlobFuse2ProtocolFuse, nil } return "", fmt.Errorf("unknown volume mount type %s", radixVolumeMount.Type) } @@ -203,7 +199,7 @@ func getCsiRadixVolumeTypeId(radixVolumeMount *radixv1.RadixVolumeMount) (string // GetVolumes Get volumes of a component by RadixVolumeMounts func GetVolumes(ctx context.Context, kubeclient kubernetes.Interface, kubeutil *kube.Kube, namespace string, environment string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { var volumes []corev1.Volume - volumeMountVolumes, err := getComponentVolumeMountVolumes(ctx, kubeclient, namespace, environment, deployComponent, existingVolumes) + volumeMountVolumes, err := getComponentVolumeMountVolumes(deployComponent, existingVolumes) if err != nil { return nil, err } @@ -270,58 +266,84 @@ func getComponentSecretRefsAzureKeyVaultVolumes(ctx context.Context, kubeutil *k return volumes, nil } -func getComponentVolumeMountVolumes(ctx context.Context, kubeClient kubernetes.Interface, namespace, environment string, deployComponent radixv1.RadixCommonDeployComponent, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { +func getComponentVolumeMountVolumes(deployComponent radixv1.RadixCommonDeployComponent, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { componentName := deployComponent.GetName() - existingVolumesMap := getVolumesMap(existingVolumes) + existingVolumeSourcesMap := getVolumesSourcesByVolumeNamesMap(existingVolumes) var volumes []corev1.Volume - for _, volumeMount := range deployComponent.GetVolumeMounts() { - volumeName, err := getVolumeMountVolumeName(&volumeMount, componentName) + var errs []error + for _, radixVolumeMount := range deployComponent.GetVolumeMounts() { + volume, err := createVolume(radixVolumeMount, componentName, existingVolumeSourcesMap) if err != nil { - return nil, err - } - var existingVolumeSource *corev1.VolumeSource - if existingVolume, ok := existingVolumesMap[volumeName]; ok { - existingVolumeSource = &existingVolume.VolumeSource - } - volumeSource, err := getVolumeSource(ctx, kubeClient, namespace, environment, componentName, &volumeMount, existingVolumeSource) - if err != nil { - return nil, err + errs = append(errs, err) + continue } - volumes = append(volumes, corev1.Volume{ - Name: volumeName, - VolumeSource: *volumeSource, - }) + volumes = append(volumes, *volume) + } + if len(errs) > 0 { + return nil, errors.Join(errs...) } return volumes, nil } -func getVolumesMap(volumes []corev1.Volume) map[string]corev1.Volume { - return slice.Reduce(volumes, make(map[string]corev1.Volume), func(acc map[string]corev1.Volume, volume corev1.Volume) map[string]corev1.Volume { +func createVolume(radixVolumeMount radixv1.RadixVolumeMount, componentName string, existingVolumeSourcesMap map[string]corev1.VolumeSource) (*corev1.Volume, error) { + volumeName, err := getVolumeMountVolumeName(&radixVolumeMount, componentName) + if err != nil { + return nil, err + } + volumeSource, err := getOrCreateVolumeSource(volumeName, componentName, radixVolumeMount, existingVolumeSourcesMap) + if err != nil { + return nil, err + } + return &corev1.Volume{ + Name: volumeName, + VolumeSource: *volumeSource, + }, nil +} + +func getOrCreateVolumeSource(volumeName string, componentName string, radixVolumeMount radixv1.RadixVolumeMount, existingVolumeSourcesMap map[string]corev1.VolumeSource) (*corev1.VolumeSource, error) { + if existingVolumeSource, ok := existingVolumeSourcesMap[volumeName]; ok { + return &existingVolumeSource, nil + } + return getVolumeSource(componentName, &radixVolumeMount) +} + +func getVolumesSourcesByVolumeNamesMap(volumes []corev1.Volume) map[string]corev1.VolumeSource { + return slice.Reduce(volumes, make(map[string]corev1.VolumeSource), func(acc map[string]corev1.VolumeSource, volume corev1.Volume) map[string]corev1.VolumeSource { if volume.PersistentVolumeClaim != nil { - acc[volume.Name] = volume + acc[volume.Name] = volume.VolumeSource } return acc }) } -func getVolumeSource(ctx context.Context, kubeClient kubernetes.Interface, namespace, environment, componentName string, volumeMount *radixv1.RadixVolumeMount, existingVolumeSource *corev1.VolumeSource) (*corev1.VolumeSource, error) { +func getVolumeSource(componentName string, volumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { switch { case volumeMount.HasDeprecatedVolume(): - return getComponentVolumeMountDeprecatedVolumeSource(ctx, kubeClient, namespace, environment, componentName, volumeMount, existingVolumeSource) + return getComponentVolumeMountDeprecatedVolumeSource(componentName, volumeMount) case volumeMount.HasBlobFuse2(): - return getCsiAzureVolumeSource(ctx, kubeClient, namespace, componentName, volumeMount, existingVolumeSource) + return getCsiAzureVolumeSource(componentName, volumeMount) case volumeMount.HasEmptyDir(): return getComponentVolumeMountEmptyDirVolumeSource(volumeMount.EmptyDir), nil } return nil, fmt.Errorf("missing configuration for volumeMount %s", volumeMount.Name) } -func getComponentVolumeMountDeprecatedVolumeSource(ctx context.Context, kubeClient kubernetes.Interface, namespace, environment, componentName string, volumeMount *radixv1.RadixVolumeMount, existingVolumeSource *corev1.VolumeSource) (*corev1.VolumeSource, error) { - switch volumeMount.Type { - case radixv1.MountTypeBlobFuse2FuseCsiAzure: - return getCsiAzureVolumeSource(ctx, kubeClient, namespace, componentName, volumeMount, existingVolumeSource) +func getCsiAzureVolumeSource(componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { + pvcName, err := getCsiAzurePersistentVolumeClaimName(componentName, radixVolumeMount) + if err != nil { + return nil, err } + return &corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + }, + }, nil +} +func getComponentVolumeMountDeprecatedVolumeSource(componentName string, volumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { + if volumeMount.Type == radixv1.MountTypeBlobFuse2FuseCsiAzure { + return getCsiAzureVolumeSource(componentName, volumeMount) + } return nil, fmt.Errorf("unsupported volume type %s", volumeMount.Type) } @@ -333,32 +355,6 @@ func getComponentVolumeMountEmptyDirVolumeSource(spec *radixv1.RadixEmptyDirVolu } } -func getCsiAzureVolumeSource(ctx context.Context, kubeClient kubernetes.Interface, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, existingVolumeSource *corev1.VolumeSource) (*corev1.VolumeSource, error) { - existingNotTerminatingPvcList, err := getPvcNotTerminating(ctx, kubeClient, namespace, componentName, radixVolumeMount) - if err != nil { - return nil, err - } - pvList, err := getFunctionalCsiPersistentVolumesForNamespace(ctx, kubeClient, namespace) - if err != nil { - return nil, err - } - - var pvcName string - if existingNotTerminatingPvcList != nil { - pvcName = existingNotTerminatingPvcList.Name - } else { - pvcName, err = createCsiAzurePersistentVolumeClaimName(componentName, radixVolumeMount) - if err != nil { - return nil, err - } - } - return &corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvcName, - }, - }, nil -} - func getVolumeMountVolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { switch { case volumeMount.HasDeprecatedVolume(): @@ -366,7 +362,6 @@ func getVolumeMountVolumeName(volumeMount *radixv1.RadixVolumeMount, componentNa case volumeMount.HasBlobFuse2(): return getCsiAzureVolumeMountName(componentName, volumeMount) } - return fmt.Sprintf("radix-vm-%s", volumeMount.Name), nil } @@ -378,7 +373,7 @@ func getVolumeMountDeprecatedVolumeName(volumeMount *radixv1.RadixVolumeMount, c return "", fmt.Errorf("unsupported type %s", volumeMount.Type) } -func getPvcNotTerminating(ctx context.Context, kubeclient kubernetes.Interface, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { +func getFunctionalPvcList(ctx context.Context, kubeclient kubernetes.Interface, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) ([]corev1.PersistentVolumeClaim, error) { pvcList, err := kubeclient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ LabelSelector: getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentVolumeMount(componentName, radixVolumeMount.Name), }) @@ -386,20 +381,14 @@ func getPvcNotTerminating(ctx context.Context, kubeclient kubernetes.Interface, return nil, err } existingPvcs := sortPvcsByCreatedTimestampDesc(pvcList.Items) - if len(existingPvcs) == 0 { - return nil, nil - } + return slice.FindAll(existingPvcs, func(pvc corev1.PersistentVolumeClaim) bool { return pvcIsFunctional(pvc) }), nil +} - for _, pvc := range existingPvcs { - switch pvc.Status.Phase { - case corev1.ClaimPending, corev1.ClaimBound: - return &pvc, nil - } - } - return nil, nil +func pvcIsFunctional(pvc corev1.PersistentVolumeClaim) bool { + return slice.Any(functionalPersistentVolumeClaimPhases, func(phase corev1.PersistentVolumeClaimPhase) bool { return pvc.Status.Phase == phase }) } -func createCsiAzurePersistentVolumeClaimName(componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { +func getCsiAzurePersistentVolumeClaimName(componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { volumeName, err := getCsiAzureVolumeMountName(componentName, radixVolumeMount) if err != nil { return "", err @@ -488,8 +477,12 @@ func getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentVolumeMount(co return fmt.Sprintf("%s=%s, %s=%s", kube.RadixComponentLabel, componentName, kube.RadixVolumeMountNameLabel, radixVolumeMountName) } -func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appName, namespace, componentName, pvcName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { - pvc := &corev1.PersistentVolumeClaim{ +func buildPersistentVolumeClaim(appName, namespace, componentName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { + pvcName, err := getCsiAzurePersistentVolumeClaimName(componentName, radixVolumeMount) + if err != nil { + return nil, err + } + return &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, Namespace: namespace, @@ -506,10 +499,9 @@ func (deploy *Deployment) createPersistentVolumeClaim(ctx context.Context, appNa Requests: corev1.ResourceList{corev1.ResourceStorage: getVolumeCapacity(radixVolumeMount)}, }, VolumeName: pvName, - StorageClassName: pointers.Ptr(""), // avoid to use the "default" storage class + StorageClassName: pointers.Ptr(""), // use "" to avoid to use the "default" storage class }, - } - return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) + }, nil } func getVolumeCapacity(radixVolumeMount *radixv1.RadixVolumeMount) resource.Quantity { @@ -520,37 +512,33 @@ func getVolumeCapacity(radixVolumeMount *radixv1.RadixVolumeMount) resource.Quan return requestsVolumeMountSize } -func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, appName, volumeRootMount, namespace, componentName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount, identity *radixv1.Identity) error { +func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, appName, namespace, componentName, pvName, pvcName string, radixVolumeMount *radixv1.RadixVolumeMount, identity *radixv1.Identity) *corev1.PersistentVolume { identityClientId := getIdentityClientId(identity) useAzureIdentity := getUseAzureIdentity(identity, radixVolumeMount.UseAzureIdentity) csiVolumeCredSecretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, radixVolumeMount.Name) persistentVolume.ObjectMeta.Name = pvName persistentVolume.ObjectMeta.Labels = getCsiAzurePersistentVolumeLabels(appName, namespace, componentName, radixVolumeMount) persistentVolume.ObjectMeta.Annotations = getCsiAzurePersistentVolumeAnnotations(csiVolumeCredSecretName, namespace, useAzureIdentity) - mountOptions, err := getCsiAzurePersistentVolumeMountOptions(volumeRootMount, namespace, componentName, radixVolumeMount) - if err != nil { - return err - } persistentVolume.Spec.StorageClassName = "" - persistentVolume.Spec.MountOptions = mountOptions + persistentVolume.Spec.MountOptions = getCsiAzurePersistentVolumeMountOptionsForAzureBlob(radixVolumeMount) persistentVolume.Spec.Capacity = corev1.ResourceList{corev1.ResourceStorage: getVolumeCapacity(radixVolumeMount)} persistentVolume.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{getVolumeMountAccessMode(radixVolumeMount)} persistentVolume.Spec.ClaimRef = &corev1.ObjectReference{ APIVersion: "v1", Kind: k8s.KindPersistentVolumeClaim, Namespace: namespace, - Name: pvName, + Name: pvcName, } persistentVolume.Spec.CSI = &corev1.CSIPersistentVolumeSource{ Driver: provisionerBlobCsiAzure, VolumeHandle: getVolumeHandle(namespace, componentName, pvName, radixVolumeMount), - VolumeAttributes: getCsiAzurePersistentVolumeAttributes(radixVolumeMount, pvName, namespace, useAzureIdentity, identityClientId), + VolumeAttributes: getCsiAzurePersistentVolumeAttributes(namespace, radixVolumeMount, pvName, pvcName, useAzureIdentity, identityClientId), } if !useAzureIdentity { persistentVolume.Spec.CSI.NodeStageSecretRef = &corev1.SecretReference{Name: csiVolumeCredSecretName, Namespace: namespace} } persistentVolume.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimRetain. PersistentVolumeReclaimPolicy deletes volume on unmount. - return nil + return persistentVolume } // Specify a value the driver can use to uniquely identify the share in the cluster. @@ -590,7 +578,7 @@ func getCsiAzurePersistentVolumeLabels(appName, namespace, componentName string, } } -func getCsiAzurePersistentVolumeAttributes(radixVolumeMount *radixv1.RadixVolumeMount, pvName, namespace string, useAzureIdentity bool, clientId string) map[string]string { +func getCsiAzurePersistentVolumeAttributes(namespace string, radixVolumeMount *radixv1.RadixVolumeMount, pvName, pvcName string, useAzureIdentity bool, clientId string) map[string]string { attributes := make(map[string]string) switch GetCsiAzureVolumeMountType(radixVolumeMount) { case radixv1.MountTypeBlobFuse2FuseCsiAzure: @@ -608,7 +596,7 @@ func getCsiAzurePersistentVolumeAttributes(radixVolumeMount *radixv1.RadixVolume } } attributes[csiVolumeMountAttributePvName] = pvName - attributes[csiVolumeMountAttributePvcName] = pvName + attributes[csiVolumeMountAttributePvcName] = pvcName attributes[csiVolumeMountAttributePvcNamespace] = namespace if !useAzureIdentity { attributes[csiVolumeMountAttributeSecretNamespace] = namespace @@ -618,16 +606,7 @@ func getCsiAzurePersistentVolumeAttributes(radixVolumeMount *radixv1.RadixVolume return attributes } -func getCsiAzurePersistentVolumeMountOptions(volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) ([]string, error) { - csiVolumeTypeId, err := getCsiRadixVolumeTypeId(radixVolumeMount) - if err != nil { - return nil, err - } - tmpPath := fmt.Sprintf(csiVolumeNodeMountPathTemplate, volumeRootMount, namespace, csiVolumeTypeId, componentName, radixVolumeMount.Name, getRadixVolumeMountStorage(radixVolumeMount)) - return getCsiAzurePersistentVolumeMountOptionsForAzureBlob(tmpPath, radixVolumeMount) -} - -func getCsiAzurePersistentVolumeMountOptionsForAzureBlob(tmpPath string, radixVolumeMount *radixv1.RadixVolumeMount) ([]string, error) { +func getCsiAzurePersistentVolumeMountOptionsForAzureBlob(radixVolumeMount *radixv1.RadixVolumeMount) []string { mountOptions := []string{ // fmt.Sprintf("--%s=%s", csiPersistentVolumeTmpPathMountOption, tmpPath),//TODO fix this path to be able to mount on external mount "--file-cache-timeout-in-seconds=120", @@ -654,7 +633,7 @@ func getCsiAzurePersistentVolumeMountOptionsForAzureBlob(tmpPath string, radixVo mountOptions = append(mountOptions, getStreamingMountOptions(radixVolumeMount.BlobFuse2.Streaming)...) mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeUseAdlsMountOption, radixVolumeMount.BlobFuse2.UseAdls != nil && *radixVolumeMount.BlobFuse2.UseAdls)) } - return mountOptions, nil + return mountOptions } func getStreamingMountOptions(streaming *radixv1.RadixVolumeMountStreaming) []string { @@ -773,10 +752,11 @@ func getRadixVolumeMountStorage(radixVolumeMount *radixv1.RadixVolumeMount) stri } func (deploy *Deployment) garbageCollectOrphanedCsiAzurePersistentVolumes(ctx context.Context, excludePvcNames map[string]any) error { - pvList, err := deploy.getPersistentVolumesForPvc(ctx) + pvList, err := deploy.kubeclient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) if err != nil { return err } + var errs []error for _, pv := range pvList.Items { if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.Kind != k8s.KindPersistentVolumeClaim || !knownCSIDriver(pv.Spec.CSI) { continue @@ -788,10 +768,13 @@ func (deploy *Deployment) garbageCollectOrphanedCsiAzurePersistentVolumes(ctx co continue } log.Ctx(ctx).Info().Msgf("Delete orphaned Csi Azure PersistantVolume %s of PersistantVolumeClaim %s", pv.Name, pv.Spec.ClaimRef.Name) - if err = deploy.deletePersistentVolume(ctx, pv.Name); err != nil { - return err + if err := deploy.deletePersistentVolume(ctx, pv.Name); err != nil && !k8serrors.IsNotFound(err) { + errs = append(errs, err) } } + if len(errs) > 0 { + return errors.Join(errs...) + } return nil } @@ -804,60 +787,100 @@ func knownCSIDriver(csiPersistentVolumeSource *corev1.CSIPersistentVolumeSource) } // createOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims, PersistentVolume -func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Context, desiredDeployment *appsv1.Deployment, deployComponent radixv1.RadixCommonDeployComponent) error { - namespace := deploy.radixDeployment.GetNamespace() - appName := deploy.radixDeployment.Spec.AppName +func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Context, deployComponent radixv1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) error { componentName := desiredDeployment.ObjectMeta.Name - volumeRootMount := "/tmp" // TODO: add to environment variable, so this volume can be mounted to external disk - pvList, err := getFunctionalCsiPersistentVolumesForNamespace(ctx, deploy.kubeclient, namespace) - if err != nil { + if err := deploy.createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx, componentName, deployComponent.GetIdentity(), desiredDeployment); err != nil { return err } - pvcList, err := deploy.getCsiAzurePersistentVolumeClaims(ctx, namespace, componentName) + currentlyUsedPvcNames, err := deploy.getCurrentlyUsedPersistentVolumeClaims(ctx) if err != nil { return err } + if err = deploy.garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx, componentName, currentlyUsedPvcNames); err != nil { + return err + } + return deploy.garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, currentlyUsedPvcNames) +} - existingPvcMap := internal.GetPersistentVolumeClaimMap(&pvcList.Items, false) - radixVolumeMountMap := deploy.getRadixVolumeMountMapByCsiAzureVolumeMountName(componentName) - var actualPersistentVolumeNames []string - actualPvcNames, err := deploy.getCurrentlyUsedPersistentVolumeClaims(ctx, namespace) +func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, componentName string, identity *radixv1.Identity, desiredDeployment *appsv1.Deployment) error { + namespace := deploy.radixDeployment.GetNamespace() + functionalPvList, err := getFunctionalCsiPersistentVolumesForNamespace(ctx, deploy.kubeclient, namespace) + if err != nil { + return err + } + pvcByNameMap, err := deploy.getPvcByNameMap(ctx, namespace, componentName) if err != nil { return err } - identity := deployComponent.GetIdentity() + radixVolumeMountsByNameMap := deploy.getRadixVolumeMountsByNameMap(componentName) + var errs []error + var volumes []corev1.Volume for _, volume := range desiredDeployment.Spec.Template.Spec.Volumes { - if volume.PersistentVolumeClaim == nil { - continue - } - radixVolumeMount, existsRadixVolumeMount := radixVolumeMountMap[volume.Name] + radixVolumeMount, existsRadixVolumeMount := radixVolumeMountsByNameMap[volume.Name] if !existsRadixVolumeMount { - return fmt.Errorf("not found Radix volume mount for desired volume %s", volume.Name) + continue } - pv, pvIsCreated, err := deploy.getOrCreateCsiAzurePersistentVolume(ctx, appName, volumeRootMount, namespace, componentName, radixVolumeMount, pvList, volume.PersistentVolumeClaim.ClaimName, identity) + processedVolume, err := deploy.createOrUpdateCsiAzureVolumeResourcesForVolume(ctx, namespace, componentName, identity, volume, radixVolumeMount, functionalPvList, pvcByNameMap) if err != nil { - return err + errs = append(errs, err) + continue } - pvc, err := deploy.createCsiAzurePersistentVolumeClaim(ctx, pv, pvIsCreated, appName, namespace, componentName, radixVolumeMount, volume.PersistentVolumeClaim.ClaimName, existingPvcMap) - if err != nil { - return err + if processedVolume != nil { + volumes = append(volumes, *processedVolume) } - actualPersistentVolumeNames = append(actualPersistentVolumeNames, pv.Name) - volume.PersistentVolumeClaim.ClaimName = pvc.Name - actualPvcNames[pvc.Name] = struct{}{} } - err = deploy.garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx, namespace, pvcList, actualPvcNames) + if len(errs) > 0 { + return errors.Join(errs...) + } + desiredDeployment.Spec.Template.Spec.Volumes = volumes + return nil +} + +func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, namespace string, componentName string, identity *radixv1.Identity, volume corev1.Volume, radixVolumeMount *radixv1.RadixVolumeMount, functionalPvList []corev1.PersistentVolume, pvcByNameMap map[string]*corev1.PersistentVolumeClaim) (*corev1.Volume, error) { + if volume.PersistentVolumeClaim == nil || volume.CSI == nil { + return &volume, nil + } + appName := deploy.radixDeployment.Spec.AppName + pvcName := volume.PersistentVolumeClaim.ClaimName + pvName, actualPvExists := getActualExistingCsiAzurePvNameByPvcName(appName, namespace, componentName, radixVolumeMount, functionalPvList, pvcName, identity) + if !actualPvExists { + pvName = getCsiAzurePersistentVolumeName() + } + existingPvc, pvcExist := pvcByNameMap[pvcName] + newPvc, err := buildPersistentVolumeClaim(appName, namespace, componentName, pvName, radixVolumeMount) if err != nil { - return err + return nil, err } - err = deploy.garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, actualPvcNames) - if err != nil && !k8serrors.IsNotFound(err) { - return err + if pvcExist && !internal.EqualPersistentVolumeClaims(existingPvc, newPvc) { + pvcName = newPvc.GetName() } - return nil + if !actualPvExists { + log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) + pv := populateCsiAzurePersistentVolume(&corev1.PersistentVolume{}, appName, namespace, componentName, pvName, pvcName, radixVolumeMount, identity) + if _, err = deploy.kubeclient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil { + return nil, err + } + } + if !pvcExist { + log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for PersistentVolume %s", pvcName, namespace, pvName) + if _, err := deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, newPvc, metav1.CreateOptions{}); err != nil { + return nil, err + } + } + volume.PersistentVolumeClaim.ClaimName = pvcName + return &volume, nil } -func (deploy *Deployment) getCurrentlyUsedPersistentVolumeClaims(ctx context.Context, namespace string) (map[string]any, error) { +func (deploy *Deployment) getPvcByNameMap(ctx context.Context, namespace string, componentName string) (map[string]*corev1.PersistentVolumeClaim, error) { + pvcList, err := deploy.getCsiAzurePersistentVolumeClaims(ctx, namespace, componentName) + if err != nil { + return nil, err + } + return internal.GetPersistentVolumeClaimMap(&pvcList.Items, false), nil +} + +func (deploy *Deployment) getCurrentlyUsedPersistentVolumeClaims(ctx context.Context) (map[string]any, error) { + namespace := deploy.radixDeployment.GetNamespace() pvcNames := make(map[string]any) deploymentList, err := deploy.kubeclient.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) if err != nil { @@ -884,92 +907,62 @@ func addUsedPersistenceVolumeClaimsFrom(podTemplate corev1.PodTemplateSpec, pvcM } } -func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx context.Context, namespace string, pvcList *corev1.PersistentVolumeClaimList, excludePvcNames map[string]any) error { +func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx context.Context, componentName string, excludePvcNames map[string]any) error { + namespace := deploy.radixDeployment.GetNamespace() + pvcList, err := deploy.getCsiAzurePersistentVolumeClaims(ctx, namespace, componentName) + if err != nil { + return err + } + var errs []error for _, pvc := range pvcList.Items { if _, ok := excludePvcNames[pvc.Name]; ok { continue } - pvName := pvc.Spec.VolumeName log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolumeClaim %s in namespace %s", pvc.Name, namespace) if err := deploy.deletePersistentVolumeClaim(ctx, namespace, pvc.Name); err != nil { - return err + errs = append(errs, err) + continue } + pvName := pvc.Spec.VolumeName log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolume %s in namespace %s", pvName, namespace) if err := deploy.deletePersistentVolume(ctx, pvName); err != nil { - return err - } - } - return nil -} - -func (deploy *Deployment) createCsiAzurePersistentVolumeClaim(ctx context.Context, pv *corev1.PersistentVolume, requiredNewPvc bool, appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumeClaimName string, pvcMap map[string]*corev1.PersistentVolumeClaim) (*corev1.PersistentVolumeClaim, error) { - if pvc, ok := pvcMap[persistentVolumeClaimName]; ok { - if len(pvc.Spec.VolumeName) == 0 { - return pvc, nil - } - if !requiredNewPvc && strings.EqualFold(pvc.Spec.VolumeName, pv.Name) { - return pvc, nil + errs = append(errs, err) } - - log.Ctx(ctx).Debug().Msgf("Delete in garbage-collect an old PersistentVolumeClaim %s in namespace %s: changed PersistentVolume name to %s", pvc.Name, namespace, pv.Name) } - persistentVolumeClaimName, err := createCsiAzurePersistentVolumeClaimName(componentName, radixVolumeMount) - if err != nil { - return nil, err + if len(errs) > 0 { + return errors.Join(errs...) } - log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for PersistentVolume %s", persistentVolumeClaimName, namespace, pv.Name) - return deploy.createPersistentVolumeClaim(ctx, appName, namespace, componentName, persistentVolumeClaimName, pv.Name, radixVolumeMount) + return nil } -func (deploy *Deployment) getOrCreateCsiAzurePersistentVolume(ctx context.Context, appName, volumeRootMount, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, pvList []corev1.PersistentVolume, pvcName string, identity *radixv1.Identity) (*corev1.PersistentVolume, bool, error) { - - for _, existingPv := range pvList { +func getActualExistingCsiAzurePvNameByPvcName(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, existingPvList []corev1.PersistentVolume, pvcName string, identity *radixv1.Identity) (string, bool) { + for _, existingPv := range existingPvList { if existingPv.Spec.ClaimRef != nil && existingPv.Spec.ClaimRef.Name == pvcName { - desiredPv := existingPv.DeepCopy() - err := populateCsiAzurePersistentVolume(desiredPv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, identity) - if err != nil { - return nil, false, err - } - if equal, err := internal.EqualPersistentVolumes(&existingPv, desiredPv); equal || err != nil { - return &existingPv, false, err + desiredPv := populateCsiAzurePersistentVolume(existingPv.DeepCopy(), appName, namespace, componentName, existingPv.GetName(), pvcName, radixVolumeMount, identity) + if internal.EqualPersistentVolumes(&existingPv, desiredPv) { + return existingPv.GetName(), true } - - log.Ctx(ctx).Info().Msgf("Delete PersistentVolume %s in namespace %s", existingPv.Name, namespace) - if err = deploy.deletePersistentVolume(ctx, existingPv.Name); err != nil { - return nil, false, err - } - continue - } - - pvName := getCsiAzurePersistentVolumeName() - log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) - newPv := &corev1.PersistentVolume{} - err := populateCsiAzurePersistentVolume(newPv, appName, volumeRootMount, namespace, componentName, pvName, radixVolumeMount, identity) - if err != nil { - return nil, false, err } - desiredPersistentVolume, err := deploy.kubeclient.CoreV1().PersistentVolumes().Create(ctx, newPv, metav1.CreateOptions{}) - return desiredPersistentVolume, true, err } - return nil, false, fmt.Errorf("failed to create PersistentVolume %s in namespace %s", pvName, namespace) // TODO check if this is correct + return "", false } -func (deploy *Deployment) getRadixVolumeMountMapByCsiAzureVolumeMountName(componentName string) map[string]*radixv1.RadixVolumeMount { - volumeMountMap := make(map[string]*radixv1.RadixVolumeMount) +func (deploy *Deployment) getRadixVolumeMountsByNameMap(componentName string) map[string]*radixv1.RadixVolumeMount { + volumeMountsByNameMap := make(map[string]*radixv1.RadixVolumeMount) for _, component := range deploy.radixDeployment.Spec.Components { - if findCsiAzureVolumeForComponent(volumeMountMap, component.VolumeMounts, componentName, &component) { + if findCsiAzureVolumeForComponent(volumeMountsByNameMap, component.VolumeMounts, componentName, &component) { break } } for _, component := range deploy.radixDeployment.Spec.Jobs { - if findCsiAzureVolumeForComponent(volumeMountMap, component.VolumeMounts, componentName, &component) { + if findCsiAzureVolumeForComponent(volumeMountsByNameMap, component.VolumeMounts, componentName, &component) { break } } - return volumeMountMap + return volumeMountsByNameMap } -func findCsiAzureVolumeForComponent(volumeMountMap map[string]*radixv1.RadixVolumeMount, volumeMounts []radixv1.RadixVolumeMount, componentName string, component radixv1.RadixCommonDeployComponent) bool { +func findCsiAzureVolumeForComponent(volumeMountsByNameMap map[string]*radixv1.RadixVolumeMount, volumeMounts []radixv1.RadixVolumeMount, componentName string, component radixv1.RadixCommonDeployComponent) bool { if !strings.EqualFold(componentName, component.GetName()) { return false } @@ -982,7 +975,7 @@ func findCsiAzureVolumeForComponent(volumeMountMap map[string]*radixv1.RadixVolu if err != nil { return false } - volumeMountMap[volumeMountName] = &radixVolumeMount + volumeMountsByNameMap[volumeMountName] = &radixVolumeMount } return true } diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 7b33a8715..0200a8c41 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -267,24 +267,6 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { assert.Contains(t, volume.PersistentVolumeClaim.ClaimName, scenario.expectedPvcNamePrefix) } }) - suite.T().Run("CSI Azure and Blobfuse-flex volumes", func(t *testing.T) { - t.Parallel() - testEnv := getTestEnv() - for _, scenario := range append(scenarios, blobFuseScenario) { - component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) - assert.Nil(t, err) - assert.Len(t, volumes, 1) - volume := volumes[0] - if scenario.expectedVolumeNameIsPrefix { - assert.True(t, strings.HasPrefix(volume.Name, scenario.expectedVolumeName)) - } else { - assert.Equal(t, scenario.expectedVolumeName, volume.Name) - } - assert.Less(t, len(volume.Name), 64) - assert.Equal(t, len(scenario.expectedPvcNamePrefix) > 0, volume.PersistentVolumeClaim != nil) - } - }) suite.T().Run("Unsupported volume type", func(t *testing.T) { t.Parallel() testEnv := getTestEnv() @@ -901,7 +883,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // action deployComponent := deployment.radixDeployment.Spec.Components[0] - err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), desiredDeployment, &deployComponent) + err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), &deployComponent, desiredDeployment) assert.Nil(t, err) existingPvcs, existingScs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment) @@ -1454,7 +1436,7 @@ func equalPersistentVolumeLists(list1, list2 *[]corev1.PersistentVolume) (bool, if !ok { return false, fmt.Errorf("PersistentVolume not found by name %s in second list", pvName) } - if equal, err := internal.EqualPersistentVolumes(pv1, pv2); err != nil || !equal { + if equal := internal.EqualPersistentVolumes(pv1, pv2); err != nil || !equal { return false, err } } diff --git a/pkg/apis/internal/persistentvolume.go b/pkg/apis/internal/persistentvolume.go index 851e07fed..104e6455c 100644 --- a/pkg/apis/internal/persistentvolume.go +++ b/pkg/apis/internal/persistentvolume.go @@ -19,45 +19,45 @@ func GetPersistentVolumeMap(pvList *[]corev1.PersistentVolume) map[string]*corev } // EqualPersistentVolumes Compare two PersistentVolumes -func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) (bool, error) { +func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { // Ignore for now, due to during transition period this would affect existing volume mounts, managed by a provisioner // if !utils.EqualStringMaps(pv1.GetLabels(), pv2.GetLabels()) { // return false, nil // } - if !utils.EqualStringMaps(getAnnotations(pv1), getAnnotations(pv2)) { - return false, nil + if !utils.EqualStringMaps(getPvAnnotations(pv1), getPvAnnotations(pv2)) { + return false } if !utils.EqualStringMaps(pv1.Spec.CSI.VolumeAttributes, pv2.Spec.CSI.VolumeAttributes) { - return false, nil + return false } if !utils.EqualStringMaps(getMountOptionsMap(pv1.Spec.MountOptions), getMountOptionsMap(pv2.Spec.MountOptions)) { - return false, nil + return false } - if pv1.Spec.StorageClassName != pv2.Spec.StorageClassName || - pv1.Spec.Capacity[corev1.ResourceStorage] != pv2.Spec.Capacity[corev1.ResourceStorage] || + // ignore pv1.Spec.StorageClassName != pv2.Spec.StorageClassName for transition period + if pv1.Spec.Capacity[corev1.ResourceStorage] != pv2.Spec.Capacity[corev1.ResourceStorage] || len(pv1.Spec.AccessModes) != len(pv2.Spec.AccessModes) || (len(pv1.Spec.AccessModes) != 1 && pv1.Spec.AccessModes[0] != pv2.Spec.AccessModes[0]) || pv1.Spec.CSI.Driver != pv2.Spec.CSI.Driver { - return false, nil + return false } if pv1.Spec.CSI.NodeStageSecretRef != nil { if pv2.Spec.CSI.NodeStageSecretRef == nil || pv1.Spec.CSI.NodeStageSecretRef.Name != pv2.Spec.CSI.NodeStageSecretRef.Name { - return false, nil + return false } } else if pv2.Spec.CSI.NodeStageSecretRef != nil { - return false, nil + return false } if pv1.Spec.ClaimRef != nil { if pv2.Spec.ClaimRef == nil || pv1.Spec.ClaimRef.Name != pv2.Spec.ClaimRef.Name { - return false, nil + return false } } else if pv2.Spec.ClaimRef != nil { - return false, nil + return false } - return true, nil + return true } -func getAnnotations(pv *corev1.PersistentVolume) map[string]string { +func getPvAnnotations(pv *corev1.PersistentVolume) map[string]string { annotations := make(map[string]string) for key, value := range pv.GetAnnotations() { if key == "kubectl.kubernetes.io/last-applied-configuration" { diff --git a/pkg/apis/internal/persistentvolumeclaim.go b/pkg/apis/internal/persistentvolumeclaim.go index ee105e738..5284a1afb 100644 --- a/pkg/apis/internal/persistentvolumeclaim.go +++ b/pkg/apis/internal/persistentvolumeclaim.go @@ -18,3 +18,24 @@ func GetPersistentVolumeClaimMap(pvcList *[]corev1.PersistentVolumeClaim, ignore } return pvcMap } + +// EqualPersistentVolumeClaims Compare two PersistentVolumeClaims +func EqualPersistentVolumeClaims(pvc1, pvc2 *corev1.PersistentVolumeClaim) bool { + if pvc1.GetNamespace() != pvc2.GetNamespace() { + return false + } + if !utils.EqualStringMaps(pvc1.GetAnnotations(), pvc2.GetAnnotations()) { + return false + } + if !utils.EqualStringMaps(pvc1.GetLabels(), pvc2.GetLabels()) { + return false + } + // ignore pvc1.Spec.StorageClassName != pvc2.Spec.StorageClassName for transition period + if pvc1.Spec.Resources.Requests[corev1.ResourceStorage] != pvc2.Spec.Resources.Requests[corev1.ResourceStorage] || + len(pvc1.Spec.AccessModes) != len(pvc2.Spec.AccessModes) || + (len(pvc1.Spec.AccessModes) != 1 && pvc1.Spec.AccessModes[0] != pvc2.Spec.AccessModes[0]) || + pvc1.Spec.VolumeMode != pvc2.Spec.VolumeMode { + return false + } + return true +} From cf9199acabd9a3f86464c78309d3f15cfb9f7806 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Wed, 4 Dec 2024 12:31:29 +0100 Subject: [PATCH 11/68] Fixed unit test, regenerated types --- .../templates/radixapplication.yaml | 26 ++++-- .../templates/radixdeployment.yaml | 80 ++++++++++++------ json-schema/radixapplication.json | 8 +- pkg/apis/batch/kubejob.go | 2 +- pkg/apis/deployment/kubedeployment.go | 2 +- pkg/apis/deployment/volumemount.go | 32 ++++--- pkg/apis/deployment/volumemount_test.go | 84 +++++++++++-------- 7 files changed, 145 insertions(+), 89 deletions(-) diff --git a/charts/radix-operator/templates/radixapplication.yaml b/charts/radix-operator/templates/radixapplication.yaml index 824b5a92f..58da6b84f 100644 --- a/charts/radix-operator/templates/radixapplication.yaml +++ b/charts/radix-operator/templates/radixapplication.yaml @@ -1039,8 +1039,11 @@ spec: - "" type: string storageAccount: - description: Name of a storage account. If not - defined - it will be configured in a secret. + description: Name of a storage account. It is + mandatory when using a workload identity. + It is optional when using Access Key, if it + is not defined, it will be configured in a + secret. type: string streaming: description: |- @@ -1844,8 +1847,10 @@ spec: - "" type: string storageAccount: - description: Name of a storage account. If not defined - - it will be configured in a secret. + description: Name of a storage account. It is mandatory + when using a workload identity. It is optional when + using Access Key, if it is not defined, it will + be configured in a secret. type: string streaming: description: |- @@ -2713,8 +2718,11 @@ spec: - "" type: string storageAccount: - description: Name of a storage account. If not - defined - it will be configured in a secret. + description: Name of a storage account. It is + mandatory when using a workload identity. + It is optional when using Access Key, if it + is not defined, it will be configured in a + secret. type: string streaming: description: |- @@ -3274,8 +3282,10 @@ spec: - "" type: string storageAccount: - description: Name of a storage account. If not defined - - it will be configured in a secret. + description: Name of a storage account. It is mandatory + when using a workload identity. It is optional when + using Access Key, if it is not defined, it will + be configured in a secret. type: string streaming: description: |- diff --git a/charts/radix-operator/templates/radixdeployment.yaml b/charts/radix-operator/templates/radixdeployment.yaml index 838678a13..bc44ad4cd 100644 --- a/charts/radix-operator/templates/radixdeployment.yaml +++ b/charts/radix-operator/templates/radixdeployment.yaml @@ -734,7 +734,7 @@ spec: description: |- Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce @@ -742,7 +742,9 @@ spec: - "" type: string azureFile: - description: AzureFile settings for Azure File CSI driver + description: |- + AzureFile settings for Azure File CSI driver + Deprecated. properties: accessMode: description: |- @@ -790,7 +792,7 @@ spec: description: |- Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -798,7 +800,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage FUSE - CSI driver + CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -832,7 +834,6 @@ spec: FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -840,6 +841,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. Applicable + when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -851,6 +856,12 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. It is mandatory + when using a workload identity. It is optional when + using Access Key, if it is not defined, it will + be configured in a secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -924,7 +935,7 @@ spec: gid: description: |- GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -941,33 +952,38 @@ spec: requestsStorage: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string skuName: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string storage: description: |- Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string type: description: |- Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - blob - azure-blob - - azure-file - "" type: string uid: description: |- UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Key Vault will be acquired using + Azure Workload Identity instead of using a ClientID + and Secret. + type: boolean required: - name - path @@ -1317,7 +1333,7 @@ spec: description: |- Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce @@ -1325,7 +1341,9 @@ spec: - "" type: string azureFile: - description: AzureFile settings for Azure File CSI driver + description: |- + AzureFile settings for Azure File CSI driver + Deprecated. properties: accessMode: description: |- @@ -1373,7 +1391,7 @@ spec: description: |- Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -1381,7 +1399,7 @@ spec: type: string blobFuse2: description: BlobFuse2 settings for Azure Storage FUSE - CSI driver + CSI driver with the protocol fuse2 properties: accessMode: description: |- @@ -1415,7 +1433,6 @@ spec: FUSE driver. Default is fuse2. enum: - fuse2 - - nfs - "" type: string requestsStorage: @@ -1423,6 +1440,10 @@ spec: Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim type: string + resourceGroup: + description: ResourceGroup of a storage account. Applicable + when using a workload identity. + type: string skuName: description: |- SKU Type of Azure storage. @@ -1434,6 +1455,12 @@ spec: - Standard_RAGRS - "" type: string + storageAccount: + description: Name of a storage account. It is mandatory + when using a workload identity. It is optional when + using Access Key, if it is not defined, it will + be configured in a secret. + type: string streaming: description: |- Configure Streaming mode. Used for blobfuse2. @@ -1507,7 +1534,7 @@ spec: gid: description: |- GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -1524,33 +1551,38 @@ spec: requestsStorage: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string skuName: description: |- More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string storage: description: |- Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string type: description: |- Type defines the storage type. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. enum: - blob - azure-blob - - azure-file - "" type: string uid: description: |- UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 or AzureFile instead. + Deprecated, use BlobFuse2 instead. type: string + useAzureIdentity: + description: UseAzureIdentity defines that credentials + for accessing Azure Key Vault will be acquired using + Azure Workload Identity instead of using a ClientID + and Secret. + type: boolean required: - name - path diff --git a/json-schema/radixapplication.json b/json-schema/radixapplication.json index 6eeb9ebad..49c1f38a9 100644 --- a/json-schema/radixapplication.json +++ b/json-schema/radixapplication.json @@ -1003,7 +1003,7 @@ "type": "string" }, "storageAccount": { - "description": "Name of a storage account. If not defined - it will be configured in a secret.", + "description": "Name of a storage account. It is mandatory when using a workload identity. It is optional when using Access Key, if it is not defined, it will be configured in a secret.", "type": "string" }, "streaming": { @@ -1828,7 +1828,7 @@ "type": "string" }, "storageAccount": { - "description": "Name of a storage account. If not defined - it will be configured in a secret.", + "description": "Name of a storage account. It is mandatory when using a workload identity. It is optional when using Access Key, if it is not defined, it will be configured in a secret.", "type": "string" }, "streaming": { @@ -2729,7 +2729,7 @@ "type": "string" }, "storageAccount": { - "description": "Name of a storage account. If not defined - it will be configured in a secret.", + "description": "Name of a storage account. It is mandatory when using a workload identity. It is optional when using Access Key, if it is not defined, it will be configured in a secret.", "type": "string" }, "streaming": { @@ -3299,7 +3299,7 @@ "type": "string" }, "storageAccount": { - "description": "Name of a storage account. If not defined - it will be configured in a secret.", + "description": "Name of a storage account. It is mandatory when using a workload identity. It is optional when using Access Key, if it is not defined, it will be configured in a secret.", "type": "string" }, "streaming": { diff --git a/pkg/apis/batch/kubejob.go b/pkg/apis/batch/kubejob.go index b3ab02674..c6ef65ddc 100644 --- a/pkg/apis/batch/kubejob.go +++ b/pkg/apis/batch/kubejob.go @@ -174,7 +174,7 @@ func (s *syncer) getJobPodImagePullSecrets(rd *radixv1.RadixDeployment) []corev1 } func (s *syncer) getVolumes(ctx context.Context, namespace, environment string, batchJob *radixv1.RadixBatchJob, radixJobComponent *radixv1.RadixDeployJobComponent, radixDeploymentName string) ([]corev1.Volume, error) { - volumes, err := deployment.GetVolumes(ctx, s.kubeClient, s.kubeUtil, namespace, environment, radixJobComponent, radixDeploymentName, nil) + volumes, err := deployment.GetVolumes(ctx, s.kubeUtil, namespace, radixJobComponent, radixDeploymentName, nil) if err != nil { return nil, err } diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index 2635523bb..711cda098 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -286,7 +286,7 @@ func (deploy *Deployment) setDesiredDeploymentProperties(ctx context.Context, de desiredDeployment.Spec.Template.Spec.Affinity = utils.GetAffinityForDeployComponent(ctx, deployComponent, appName, componentName) desiredDeployment.Spec.Template.Spec.Tolerations = utils.GetDeploymentPodSpecTolerations(deployComponent.GetNode()) - volumes, err := GetVolumes(ctx, deploy.kubeclient, deploy.kubeutil, deploy.getNamespace(), deploy.radixDeployment.Spec.Environment, deployComponent, deploy.radixDeployment.GetName(), desiredDeployment.Spec.Template.Spec.Volumes) + volumes, err := GetVolumes(ctx, deploy.kubeutil, deploy.getNamespace(), deployComponent, deploy.radixDeployment.GetName(), desiredDeployment.Spec.Template.Spec.Volumes) if err != nil { return err } diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index e524da116..5ff24c3c3 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -55,8 +55,8 @@ const ( csiVolumeMountAttributeResourceGroup = "resourcegroup" csiVolumeMountAttributeSecretNamespace = "secretnamespace" csiVolumeMountAttributePvName = "csi.storage.k8s.io/pv/name" - csiVolumeMountAttributePvcName = "csi.storage.k8s.io/pv/name" - csiVolumeMountAttributePvcNamespace = "csi.storage.k8s.io/pv/name" + csiVolumeMountAttributePvcName = "csi.storage.k8s.io/pvc/name" + csiVolumeMountAttributePvcNamespace = "csi.storage.k8s.io/pvc/namespace" csiVolumeMountAnnotationProvisionedBy = "pv.kubernetes.io/provisioned-by" csiVolumeMountAnnotationProvisionerDeletionSecretName = "volume.kubernetes.io/provisioner-deletion-secret-name" csiVolumeMountAnnotationProvisionerDeletionSecretNamespace = "volume.kubernetes.io/provisioner-deletion-secret-namespace" @@ -197,7 +197,7 @@ func getCsiRadixVolumeTypeId(radixVolumeMount *radixv1.RadixVolumeMount) (string } // GetVolumes Get volumes of a component by RadixVolumeMounts -func GetVolumes(ctx context.Context, kubeclient kubernetes.Interface, kubeutil *kube.Kube, namespace string, environment string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { +func GetVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { var volumes []corev1.Volume volumeMountVolumes, err := getComponentVolumeMountVolumes(deployComponent, existingVolumes) if err != nil { @@ -215,13 +215,11 @@ func GetVolumes(ctx context.Context, kubeclient kubernetes.Interface, kubeutil * } func getComponentSecretRefsVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.Volume, error) { - var volumes []corev1.Volume azureKeyVaultVolumes, err := getComponentSecretRefsAzureKeyVaultVolumes(ctx, kubeutil, namespace, deployComponent, radixDeploymentName) if err != nil { return nil, err } - volumes = append(volumes, azureKeyVaultVolumes...) - return volumes, nil + return azureKeyVaultVolumes, nil } func getComponentSecretRefsAzureKeyVaultVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.Volume, error) { @@ -370,7 +368,7 @@ func getVolumeMountDeprecatedVolumeName(volumeMount *radixv1.RadixVolumeMount, c case radixv1.MountTypeBlobFuse2FuseCsiAzure: return getCsiAzureVolumeMountName(componentName, volumeMount) } - return "", fmt.Errorf("unsupported type %s", volumeMount.Type) + return "", fmt.Errorf("unsupported volume type %s", volumeMount.Type) } func getFunctionalPvcList(ctx context.Context, kubeclient kubernetes.Interface, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) ([]corev1.PersistentVolumeClaim, error) { @@ -531,7 +529,7 @@ func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, } persistentVolume.Spec.CSI = &corev1.CSIPersistentVolumeSource{ Driver: provisionerBlobCsiAzure, - VolumeHandle: getVolumeHandle(namespace, componentName, pvName, radixVolumeMount), + VolumeHandle: getVolumeHandle(namespace, componentName, pvName, getRadixVolumeMountStorage(radixVolumeMount)), VolumeAttributes: getCsiAzurePersistentVolumeAttributes(namespace, radixVolumeMount, pvName, pvcName, useAzureIdentity, identityClientId), } if !useAzureIdentity { @@ -543,8 +541,8 @@ func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, // Specify a value the driver can use to uniquely identify the share in the cluster. // https://github.com/kubernetes-csi/csi-driver-smb/blob/master/docs/driver-parameters.md#pvpvc-usage -func getVolumeHandle(namespace, componentName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount) string { - return fmt.Sprintf("%s#%s#%s#%s", namespace, componentName, pvName, getRadixVolumeMountStorage(radixVolumeMount)) +func getVolumeHandle(namespace, componentName, pvName, storageName string) string { + return fmt.Sprintf("%s#%s#%s#%s", namespace, componentName, pvName, storageName) } func getUseAzureIdentity(identity *radixv1.Identity, useAzureIdentity *bool) bool { @@ -608,7 +606,6 @@ func getCsiAzurePersistentVolumeAttributes(namespace string, radixVolumeMount *r func getCsiAzurePersistentVolumeMountOptionsForAzureBlob(radixVolumeMount *radixv1.RadixVolumeMount) []string { mountOptions := []string{ - // fmt.Sprintf("--%s=%s", csiPersistentVolumeTmpPathMountOption, tmpPath),//TODO fix this path to be able to mount on external mount "--file-cache-timeout-in-seconds=120", "--use-attr-cache=true", "--cancel-list-on-mount-seconds=0", @@ -758,7 +755,7 @@ func (deploy *Deployment) garbageCollectOrphanedCsiAzurePersistentVolumes(ctx co } var errs []error for _, pv := range pvList.Items { - if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.Kind != k8s.KindPersistentVolumeClaim || !knownCSIDriver(pv.Spec.CSI) { + if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.Kind != k8s.KindPersistentVolumeClaim || pv.Spec.CSI == nil || !knownCSIDriver(pv.Spec.CSI.Driver) { continue } if !(pv.Status.Phase == corev1.VolumeReleased || pv.Status.Phase == corev1.VolumeFailed) { @@ -778,11 +775,8 @@ func (deploy *Deployment) garbageCollectOrphanedCsiAzurePersistentVolumes(ctx co return nil } -func knownCSIDriver(csiPersistentVolumeSource *corev1.CSIPersistentVolumeSource) bool { - if csiPersistentVolumeSource == nil { - return false - } - _, ok := csiVolumeProvisioners[csiPersistentVolumeSource.Driver] +func knownCSIDriver(driver string) bool { + _, ok := csiVolumeProvisioners[driver] return ok } @@ -816,6 +810,10 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx co var errs []error var volumes []corev1.Volume for _, volume := range desiredDeployment.Spec.Template.Spec.Volumes { + if volume.VolumeSource.CSI == nil || !knownCSIDriver(volume.VolumeSource.CSI.Driver) { + volumes = append(volumes, volume) + continue + } radixVolumeMount, existsRadixVolumeMount := radixVolumeMountsByNameMap[volume.Name] if !existsRadixVolumeMount { continue diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 0200a8c41..9070c8ba3 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -222,13 +222,12 @@ func (suite *VolumeMountTestSuite) Test_FailBlobCsiAzureVolumeMounts() { func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { namespace := "some-namespace" - environment := "some-env" componentName := "some-component" suite.T().Run("No volumes in component", func(t *testing.T) { t.Parallel() testEnv := getTestEnv() component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts().BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", nil) assert.Nil(t, err) assert.Len(t, volumes, 0) }) @@ -253,7 +252,7 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { for _, scenario := range scenarios { t.Logf("Scenario %s", scenario.name) component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", nil) assert.Nil(t, err) assert.Len(t, volumes, 1) volume := volumes[0] @@ -274,7 +273,7 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { {Type: "unsupported-type", Name: "volume1", Container: "storage1", Path: "path1"}, } component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(mounts...).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", nil) assert.Len(t, volumes, 0) assert.NotNil(t, err) assert.Equal(t, "unsupported volume type unsupported-type", err.Error()) @@ -283,7 +282,6 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { namespace := "some-namespace" - environment := "some-env" componentName := "some-component" scenarios := []pvcTestScenario{ { @@ -322,7 +320,7 @@ func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { _, _ = testEnv.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), &scenario.pvc, metav1.CreateOptions{}) component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", nil) assert.Nil(t, err) assert.Len(t, volumes, 1) assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name) @@ -338,7 +336,7 @@ func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { t.Logf("Scenario %s for volume mount type %s, PVC status phase '%v'", scenario.name, string(GetCsiAzureVolumeMountType(&scenario.radixVolumeMount)), scenario.pvc.Status.Phase) component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, &component, "", nil) + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", nil) assert.Nil(t, err) assert.Len(t, volumes, 1) assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name) @@ -385,7 +383,7 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{}) deployComponent := deployment.radixDeployment.Spec.Components[0] - volumes, err := GetVolumes(context.Background(), deployment.kubeclient, deployment.kubeutil, deployment.getNamespace(), deployment.radixDeployment.Spec.Environment, &deployComponent, deployment.radixDeployment.GetName(), nil) + volumes, err := GetVolumes(context.Background(), deployment.kubeutil, deployment.getNamespace(), &deployComponent, deployment.radixDeployment.GetName(), nil) assert.Nil(t, err) assert.Len(t, volumes, 0) @@ -402,7 +400,7 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{scenario.radixVolumeMount}) deployComponent := deployment.radixDeployment.Spec.Components[0] - volumes, err := GetVolumes(context.Background(), deployment.kubeclient, deployment.kubeutil, deployment.getNamespace(), deployment.radixDeployment.Spec.Environment, &deployComponent, deployment.radixDeployment.GetName(), nil) + volumes, err := GetVolumes(context.Background(), deployment.kubeutil, deployment.getNamespace(), &deployComponent, deployment.radixDeployment.GetName(), nil) assert.Nil(t, err) assert.Len(t, volumes, 1) @@ -518,7 +516,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), }, volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), + createTestVolume(props, func(v *corev1.Volume) {}), }, existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ @@ -555,7 +553,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }), }, volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) { + createTestVolume(props, func(v *corev1.Volume) { v.Name = scenarioProps.expectedVolumeName }), }, @@ -609,7 +607,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), }, volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), + createTestVolume(props, func(v *corev1.Volume) {}), }, existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ createRandomPvc(props, props.namespace, props.componentName), @@ -646,7 +644,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), }, volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), + createTestVolume(props, func(v *corev1.Volume) {}), }, existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ createRandomPvc(props, props.namespace, props.componentName), @@ -679,7 +677,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), }, volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), + createTestVolume(props, func(v *corev1.Volume) {}), }, existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ createRandomPvc(props, props.namespace, props.componentName), @@ -710,7 +708,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), }, volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), + createTestVolume(props, func(v *corev1.Volume) {}), }, existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ createRandomPvc(props, props.namespace, props.componentName), @@ -741,7 +739,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), }, volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), + createTestVolume(props, func(v *corev1.Volume) {}), }, existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ @@ -788,7 +786,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }), }, volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), + createTestVolume(props, func(v *corev1.Volume) {}), }, existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ @@ -841,7 +839,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }), }, volumes: []corev1.Volume{ - createVolume(props, func(v *corev1.Volume) {}), + createTestVolume(props, func(v *corev1.Volume) {}), }, existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ @@ -1020,7 +1018,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureKeyVaultResources( t.Logf("created secret provider class %s", spc.Name) } } - volumes, err := GetVolumes(context.Background(), testEnv.kubeclient, testEnv.kubeUtil, namespace, environment, radixDeployComponent, deployment.radixDeployment.GetName(), nil) + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, radixDeployComponent, deployment.radixDeployment.GetName(), nil) assert.Nil(t, err) assert.Len(t, volumes, len(scenario.expectedVolumeProps)) if len(scenario.expectedVolumeProps) == 0 { @@ -1110,7 +1108,6 @@ func Test_EmptyDir(t *testing.T) { require.NoError(t, err) assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 2) assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2) - } func createRandomPersistentVolume(props expectedPvcScProperties, namespace, componentName string) corev1.PersistentVolume { @@ -1299,7 +1296,6 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co "-o attr_timeout=120", "-o entry_timeout=120", "-o negative_timeout=120", - // fmt.Sprintf("--tmp-path=%s", props.scTmpPath), //TODO: this option does not work with blobfuse2 in some reason - investigate } idOption := getPersistentVolumeIdMountOption(props) if len(idOption) > 0 { @@ -1313,20 +1309,40 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co kube.RadixAppLabel: props.appName, kube.RadixNamespace: props.namespace, kube.RadixComponentLabel: props.componentName, - kube.RadixMountTypeLabel: string(props.radixVolumeMountType), kube.RadixVolumeMountNameLabel: props.radixVolumeMountName, }, + Annotations: map[string]string{ + csiVolumeMountAnnotationProvisionedBy: provisionerBlobCsiAzure, + csiVolumeMountAnnotationProvisionerDeletionSecretName: props.scSecretName, + csiVolumeMountAnnotationProvisionerDeletionSecretNamespace: props.namespace, + }, }, Spec: corev1.PersistentVolumeSpec{ - // Provisioner: props.scProvisioner, - // Attr: map[string]string{ - // csiPersistentVolumeProvisionerSecretNameParameter: props.scSecretName, - // csiPersistentVolumeProvisionerSecretNamespaceParameter: props.namespace, - // csiPersistentVolumeNodeStageSecretNameParameter: props.scSecretName, - // csiPersistentVolumeNodeStageSecretNamespaceParameter: props.namespace, - // }, - MountOptions: mountOptions, - // ReclaimPolicy: &reclaimPolicy, + Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: provisionerBlobCsiAzure, + VolumeHandle: getVolumeHandle(props.namespace, props.componentName, props.persistentVolumeName, props.radixStorageName), + VolumeAttributes: map[string]string{ + csiVolumeMountAttributeContainerName: props.radixStorageName, + csiVolumeMountAttributeProtocol: csiPersistentVolumeProtocolParameterFuse2, + csiVolumeMountAttributePvName: props.persistentVolumeName, + csiVolumeMountAttributePvcName: props.pvcName, + csiVolumeMountAttributePvcNamespace: props.namespace, + csiVolumeMountAttributeSecretNamespace: props.scSecretName, + // skip auto-created by the provisioner "storage.kubernetes.io/csiProvisionerIdentity": "1732528668611-2190-blob.csi.azure.com" + }, + }, + }, + AccessModes: []corev1.PersistentVolumeAccessMode{props.volumeAccessMode}, + ClaimRef: &corev1.ObjectReference{ + Namespace: props.namespace, + Name: props.pvcName, + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + StorageClassName: "", + MountOptions: mountOptions, }, } setVolumeMountAttribute(props.radixVolumeMountType, props.radixStorageName, &sc) @@ -1384,7 +1400,7 @@ func createExpectedPvc(props expectedPvcScProperties, modify func(*corev1.Persis return pvc } -func createVolume(pvcProps expectedPvcScProperties, modify func(*corev1.Volume)) corev1.Volume { +func createTestVolume(pvcProps expectedPvcScProperties, modify func(*corev1.Volume)) corev1.Volume { volume := corev1.Volume{ Name: pvcProps.volumeName, VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ @@ -1436,8 +1452,8 @@ func equalPersistentVolumeLists(list1, list2 *[]corev1.PersistentVolume) (bool, if !ok { return false, fmt.Errorf("PersistentVolume not found by name %s in second list", pvName) } - if equal := internal.EqualPersistentVolumes(pv1, pv2); err != nil || !equal { - return false, err + if equal := internal.EqualPersistentVolumes(pv1, pv2); !equal { + return false, nil } } return true, nil From 426939b7de4bb5cf46e9ecbcc9172ab5eaca8922 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 5 Dec 2024 16:07:30 +0100 Subject: [PATCH 12/68] Correcting pv, pvc logic and tests --- pkg/apis/deployment/volumemount.go | 30 +- pkg/apis/deployment/volumemount_test.go | 743 +++++++++++---------- pkg/apis/internal/persistentvolume.go | 95 ++- pkg/apis/internal/persistentvolumeclaim.go | 8 +- 4 files changed, 482 insertions(+), 394 deletions(-) diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index 5ff24c3c3..937489017 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -786,7 +786,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont if err := deploy.createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx, componentName, deployComponent.GetIdentity(), desiredDeployment); err != nil { return err } - currentlyUsedPvcNames, err := deploy.getCurrentlyUsedPersistentVolumeClaims(ctx) + currentlyUsedPvcNames, err := deploy.getCurrentlyUsedPersistentVolumeClaims(ctx, desiredDeployment) if err != nil { return err } @@ -810,7 +810,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx co var errs []error var volumes []corev1.Volume for _, volume := range desiredDeployment.Spec.Template.Spec.Volumes { - if volume.VolumeSource.CSI == nil || !knownCSIDriver(volume.VolumeSource.CSI.Driver) { + if volume.PersistentVolumeClaim == nil { volumes = append(volumes, volume) continue } @@ -835,7 +835,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx co } func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, namespace string, componentName string, identity *radixv1.Identity, volume corev1.Volume, radixVolumeMount *radixv1.RadixVolumeMount, functionalPvList []corev1.PersistentVolume, pvcByNameMap map[string]*corev1.PersistentVolumeClaim) (*corev1.Volume, error) { - if volume.PersistentVolumeClaim == nil || volume.CSI == nil { + if volume.PersistentVolumeClaim == nil { return &volume, nil } appName := deploy.radixDeployment.Spec.AppName @@ -859,8 +859,9 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolume(ctx con return nil, err } } - if !pvcExist { - log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for PersistentVolume %s", pvcName, namespace, pvName) + if !pvcExist || !internal.EqualPersistentVolumeClaims(existingPvc, newPvc) { + newPvc.SetName(pvcName) + log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for PersistentVolume %s", newPvc.GetName(), namespace, pvName) if _, err := deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, newPvc, metav1.CreateOptions{}); err != nil { return nil, err } @@ -874,10 +875,10 @@ func (deploy *Deployment) getPvcByNameMap(ctx context.Context, namespace string, if err != nil { return nil, err } - return internal.GetPersistentVolumeClaimMap(&pvcList.Items, false), nil + return internal.GetPersistentVolumeClaimMap(&pvcList.Items), nil } -func (deploy *Deployment) getCurrentlyUsedPersistentVolumeClaims(ctx context.Context) (map[string]any, error) { +func (deploy *Deployment) getCurrentlyUsedPersistentVolumeClaims(ctx context.Context, desiredDeployment *appsv1.Deployment) (map[string]any, error) { namespace := deploy.radixDeployment.GetNamespace() pvcNames := make(map[string]any) deploymentList, err := deploy.kubeclient.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) @@ -885,24 +886,25 @@ func (deploy *Deployment) getCurrentlyUsedPersistentVolumeClaims(ctx context.Con return nil, err } for _, deployment := range deploymentList.Items { - addUsedPersistenceVolumeClaimsFrom(deployment.Spec.Template, pvcNames) + pvcNames = appendUsedPersistenceVolumeClaimsFrom(pvcNames, deployment.Spec.Template.Spec.Volumes) } jobsList, err := deploy.kubeclient.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{}) if err != nil { return nil, err } for _, job := range jobsList.Items { - addUsedPersistenceVolumeClaimsFrom(job.Spec.Template, pvcNames) + pvcNames = appendUsedPersistenceVolumeClaimsFrom(pvcNames, job.Spec.Template.Spec.Volumes) } - return pvcNames, nil + return appendUsedPersistenceVolumeClaimsFrom(pvcNames, desiredDeployment.Spec.Template.Spec.Volumes), nil } -func addUsedPersistenceVolumeClaimsFrom(podTemplate corev1.PodTemplateSpec, pvcMap map[string]any) { - for _, volume := range podTemplate.Spec.Volumes { +func appendUsedPersistenceVolumeClaimsFrom(pvcMap map[string]any, volumes []corev1.Volume) map[string]any { + return slice.Reduce(volumes, pvcMap, func(acc map[string]any, volume corev1.Volume) map[string]any { if volume.PersistentVolumeClaim != nil && len(volume.PersistentVolumeClaim.ClaimName) > 0 { - pvcMap[volume.PersistentVolumeClaim.ClaimName] = struct{}{} + acc[volume.PersistentVolumeClaim.ClaimName] = struct{}{} } - } + return acc + }) } func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx context.Context, componentName string, excludePvcNames map[string]any) error { diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 9070c8ba3..d78a51fa3 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -532,345 +532,345 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { getScenario(getPropsCsiBlobVolume1Storage1(nil)), } }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - type scenarioProperties struct { - changedNewRadixVolumeName string - changedNewRadixVolumeStorageName string - expectedVolumeName string - expectedNewSecretName string - expectedNewPvcName string - expectedNewPvName string - expectedNewScTmpPath string - } - getScenario := func(props expectedPvcScProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Update storage in existing volume name and storage", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.Name = scenarioProps.changedNewRadixVolumeName - vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) { - v.Name = scenarioProps.expectedVolumeName - }), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - }), - }, - existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), - }, - existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) { - sc.ObjectMeta.Name = scenarioProps.expectedNewPvName - sc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // setPersistentVolumeMountOption(sc, "--tmp-path", scenarioProps.expectedNewScTmpPath) //TODO: this option does not work with blobfuse2 in some reason - investigate to make use separate disk volume for csi volumes - setVolumeMountAttribute(props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, sc) - sc.Spec.CSI.VolumeAttributes[csiPersistentVolumeProvisionerSecretNameParameter] = scenarioProps.expectedNewSecretName - sc.Spec.CSI.VolumeAttributes[csiPersistentVolumeNodeStageSecretNameParameter] = scenarioProps.expectedNewSecretName - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - changedNewRadixVolumeName: "volume101", - changedNewRadixVolumeStorageName: "storage101", - expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - expectedNewSecretName: "some-component-volume101-csiazurecreds", - expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - expectedNewPvName: "sc-any-app-some-env-csi-az-blob-some-component-volume101-storage101", - expectedNewScTmpPath: "/tmp/any-app-some-env/csi-az-blob/some-component/volume101/storage101", - }), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - persistentVolumeForAnotherNamespace := createRandomPersistentVolume(props, utils.RandString(10), utils.RandString(10)) - persistentVolumeForAnotherComponent := createRandomPersistentVolume(props, props.namespace, utils.RandString(10)) - pvcForAnotherNamespace := createRandomPvc(props, utils.RandString(10), utils.RandString(10)) - pvcForAnotherComponent := createRandomPvc(props, props.namespace, utils.RandString(10)) - return deploymentVolumesTestScenario{ - name: "Garbage collect orphaned PVCs and PersistentVolume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - createRandomPersistentVolume(props, props.namespace, props.componentName), - persistentVolumeForAnotherNamespace, - persistentVolumeForAnotherComponent, - }, - existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), - persistentVolumeForAnotherNamespace, - persistentVolumeForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Set readonly volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - }), - }, - existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - createRandomPersistentVolume(props, props.namespace, props.componentName), - }, - existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Set ReadWriteOnce volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - }), - }, - existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - createRandomPersistentVolume(props, props.namespace, props.componentName), - }, - existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Set ReadWriteMany volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - }), - }, - existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - createRandomPersistentVolume(props, props.namespace, props.componentName), - }, - existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, - existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = []string{ - "--file-cache-timeout-in-seconds=120", - "--use-attr-cache=true", - "--cancel-list-on-mount-seconds=0", - "-o allow_other", - "-o attr_timeout=120", - "-o entry_timeout=120", - "-o negative_timeout=120", - "-o gid=1000", - "--streaming=true", - "--use-adls=false", - } - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, - existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = []string{ - "--file-cache-timeout-in-seconds=120", - "--use-attr-cache=true", - "--cancel-list-on-mount-seconds=0", - "-o allow_other", - "-o attr_timeout=120", - "-o entry_timeout=120", - "-o negative_timeout=120", - "-o gid=1000", - "--streaming=true", - "--stream-cache-mb=101", - "--block-size-mb=102", - "--buffer-size-mb=103", - "--max-buffers=104", - "--max-blocks-per-file=105", - "--use-adls=false", - } - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has disabled streaming", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - Enabled: pointers.Ptr(false), - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, - existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = []string{ - "--file-cache-timeout-in-seconds=120", - "--use-attr-cache=true", - "--cancel-list-on-mount-seconds=0", - "-o allow_other", - "-o attr_timeout=120", - "-o entry_timeout=120", - "-o negative_timeout=120", - "-o gid=1000", - "--use-adls=false", - } - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // type scenarioProperties struct { + // changedNewRadixVolumeName string + // changedNewRadixVolumeStorageName string + // expectedVolumeName string + // expectedNewSecretName string + // expectedNewPvcName string + // expectedNewPvName string + // expectedNewScTmpPath string + // } + // getScenario := func(props expectedPvcScProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Update storage in existing volume name and storage", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.Name = scenarioProps.changedNewRadixVolumeName + // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) { + // v.Name = scenarioProps.expectedVolumeName + // }), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + // }), + // }, + // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), + // }, + // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) { + // sc.ObjectMeta.Name = scenarioProps.expectedNewPvName + // sc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // // setPersistentVolumeMountOption(sc, "--tmp-path", scenarioProps.expectedNewScTmpPath) //TODO: this option does not work with blobfuse2 in some reason - investigate to make use separate disk volume for csi volumes + // setVolumeMountAttribute(props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, sc) + // sc.Spec.CSI.VolumeAttributes[csiPersistentVolumeProvisionerSecretNameParameter] = scenarioProps.expectedNewSecretName + // sc.Spec.CSI.VolumeAttributes[csiPersistentVolumeNodeStageSecretNameParameter] = scenarioProps.expectedNewSecretName + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + // changedNewRadixVolumeName: "volume101", + // changedNewRadixVolumeStorageName: "storage101", + // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + // expectedNewSecretName: "some-component-volume101-csiazurecreds", + // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + // expectedNewPvName: "pv-radixvolumemount-some-uuid", + // expectedNewScTmpPath: "/tmp/any-app-some-env/csi-az-blob/some-component/volume101/storage101", + // }), + // } + // }()...) + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // persistentVolumeForAnotherNamespace := createRandomPersistentVolume(props, utils.RandString(10), utils.RandString(10)) + // persistentVolumeForAnotherComponent := createRandomPersistentVolume(props, props.namespace, utils.RandString(10)) + // pvcForAnotherNamespace := createRandomPvc(props, utils.RandString(10), utils.RandString(10)) + // pvcForAnotherComponent := createRandomPvc(props, props.namespace, utils.RandString(10)) + // return deploymentVolumesTestScenario{ + // name: "Garbage collect orphaned PVCs and PersistentVolume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, + // }, + // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPersistentVolume(props, props.namespace, props.componentName), + // persistentVolumeForAnotherNamespace, + // persistentVolumeForAnotherComponent, + // }, + // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), + // persistentVolumeForAnotherNamespace, + // persistentVolumeForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + // }()...) + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Set readonly volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // }), + // }, + // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPersistentVolume(props, props.namespace, props.componentName), + // }, + // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + // }()...) + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteOnce volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // }), + // }, + // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPersistentVolume(props, props.namespace, props.componentName), + // }, + // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + // }()...) + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteMany volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // }), + // }, + // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPersistentVolume(props, props.namespace, props.componentName), + // }, + // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + // }()...) + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, + // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = []string{ + // "--file-cache-timeout-in-seconds=120", + // "--use-attr-cache=true", + // "--cancel-list-on-mount-seconds=0", + // "-o allow_other", + // "-o attr_timeout=120", + // "-o entry_timeout=120", + // "-o negative_timeout=120", + // "-o gid=1000", + // "--streaming=true", + // "--use-adls=false", + // } + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + // }()...) + // + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + // StreamCache: pointers.Ptr(uint64(101)), + // BlockSize: pointers.Ptr(uint64(102)), + // BufferSize: pointers.Ptr(uint64(103)), + // MaxBuffers: pointers.Ptr(uint64(104)), + // MaxBlocksPerFile: pointers.Ptr(uint64(105)), + // } + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, + // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = []string{ + // "--file-cache-timeout-in-seconds=120", + // "--use-attr-cache=true", + // "--cancel-list-on-mount-seconds=0", + // "-o allow_other", + // "-o attr_timeout=120", + // "-o entry_timeout=120", + // "-o negative_timeout=120", + // "-o gid=1000", + // "--streaming=true", + // "--stream-cache-mb=101", + // "--block-size-mb=102", + // "--buffer-size-mb=103", + // "--max-buffers=104", + // "--max-blocks-per-file=105", + // "--use-adls=false", + // } + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + // }()...) + // + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has disabled streaming", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + // Enabled: pointers.Ptr(false), + // StreamCache: pointers.Ptr(uint64(101)), + // BlockSize: pointers.Ptr(uint64(102)), + // BufferSize: pointers.Ptr(uint64(103)), + // MaxBuffers: pointers.Ptr(uint64(104)), + // MaxBlocksPerFile: pointers.Ptr(uint64(105)), + // } + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, + // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = []string{ + // "--file-cache-timeout-in-seconds=120", + // "--use-attr-cache=true", + // "--cancel-list-on-mount-seconds=0", + // "-o allow_other", + // "-o attr_timeout=120", + // "-o entry_timeout=120", + // "-o negative_timeout=120", + // "-o gid=1000", + // "--use-adls=false", + // } + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + // }()...) suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { - t.Parallel() - for _, factory := range suite.radixCommonDeployComponentFactories { + // t.Parallel() + for _, factory := range suite.radixCommonDeployComponentFactories[:1] { for _, scenario := range scenarios { t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) testEnv := getTestEnv() @@ -884,9 +884,9 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), &deployComponent, desiredDeployment) assert.Nil(t, err) - existingPvcs, existingScs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment) + existingPvcs, existingScs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment, utils.GetEnvironmentNamespace(appName, environment)) assert.Nil(t, err) - equalPvcLists, err := equalPvcLists(&scenario.existingPvcsAfterTestRun, &existingPvcs, true) + equalPvcLists, err := equalPvcLists(&scenario.existingPvcsAfterTestRun, &existingPvcs) assert.Nil(t, err) assert.True(t, equalPvcLists) equalPersistentVolumeLists, err := equalPersistentVolumeLists(&scenario.existingPersistentVolumeAfterTestRun, &existingScs) @@ -1151,7 +1151,7 @@ func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcScProperties)) expec radixVolumeMountName: "volume1", radixStorageName: "storage1", pvcName: "pvc-csi-az-blob-some-component-volume1-storage1-12345", - persistentVolumeName: "sc-any-app-some-env-csi-az-blob-some-component-volume1-storage1", + persistentVolumeName: "pv-radixvolumemount-some-uuid", radixVolumeMountType: v1.MountTypeBlobFuse2FuseCsiAzure, requestsVolumeMountSize: "1Mi", volumeAccessMode: corev1.ReadOnlyMany, // default access mode @@ -1180,7 +1180,7 @@ func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcScProperties)) radixVolumeMountName: "volume1", radixStorageName: "storage1", pvcName: "pvc-csi-blobfuse2-fuse2-some-component-volume1-storage1-12345", - persistentVolumeName: "sc-any-app-some-env-csi-blobfuse2-fuse2-some-component-volume1-storage1", + persistentVolumeName: "pv-radixvolumemount-some-uuid", radixVolumeMountType: v1.MountTypeBlobFuse2Fuse2CsiAzure, requestsVolumeMountSize: "1Mi", volumeAccessMode: corev1.ReadOnlyMany, // default access mode @@ -1206,19 +1206,19 @@ func putExistingDeploymentVolumesScenarioDataToFakeCluster(scenario *deploymentV } } -func getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment *Deployment) ([]corev1.PersistentVolumeClaim, []corev1.PersistentVolume, error) { +func getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment *Deployment, namespace string) ([]corev1.PersistentVolumeClaim, []corev1.PersistentVolume, error) { var pvcItems []corev1.PersistentVolumeClaim var pvItems []corev1.PersistentVolume - pvcList, err := deployment.kubeclient.CoreV1().PersistentVolumeClaims("").List(context.Background(), metav1.ListOptions{}) + pvcList, err := deployment.kubeclient.CoreV1().PersistentVolumeClaims(namespace).List(context.Background(), metav1.ListOptions{}) if err != nil { - return pvcItems, pvItems, err + return nil, nil, err } if pvcList != nil && pvcList.Items != nil { pvcItems = pvcList.Items } pvList, err := deployment.kubeclient.CoreV1().PersistentVolumes().List(context.Background(), metav1.ListOptions{}) if err != nil { - return pvcItems, pvItems, err + return nil, nil, err } if pvList != nil && pvList.Items != nil { pvItems = pvList.Items @@ -1301,7 +1301,6 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co if len(idOption) > 0 { mountOptions = append(mountOptions, idOption) } - // reclaimPolicy := corev1.PersistentVolumeReclaimRetain sc := corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: props.persistentVolumeName, @@ -1329,7 +1328,7 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co csiVolumeMountAttributePvName: props.persistentVolumeName, csiVolumeMountAttributePvcName: props.pvcName, csiVolumeMountAttributePvcNamespace: props.namespace, - csiVolumeMountAttributeSecretNamespace: props.scSecretName, + csiVolumeMountAttributeSecretNamespace: props.namespace, // skip auto-created by the provisioner "storage.kubernetes.io/csiProvisionerIdentity": "1732528668611-2190-blob.csi.azure.com" }, }, @@ -1341,8 +1340,9 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co APIVersion: "v1", Kind: "PersistentVolumeClaim", }, - StorageClassName: "", - MountOptions: mountOptions, + StorageClassName: "", + MountOptions: mountOptions, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, }, } setVolumeMountAttribute(props.radixVolumeMountType, props.radixStorageName, &sc) @@ -1445,33 +1445,44 @@ func equalPersistentVolumeLists(list1, list2 *[]corev1.PersistentVolume) (bool, if len(*list1) != len(*list2) { return false, fmt.Errorf("different PersistentVolume list sizes: %v, %v", len(*list1), len(*list2)) } - map1 := internal.GetPersistentVolumeMap(list1) - map2 := internal.GetPersistentVolumeMap(list2) - for pvName, pv1 := range map1 { - pv2, ok := map2[pvName] - if !ok { - return false, fmt.Errorf("PersistentVolume not found by name %s in second list", pvName) + for _, pv1 := range *list1 { + var hasEqualPv bool + for _, pv2 := range *list2 { + if internal.EqualPersistentVolumesForTest(&pv1, &pv2) { + hasEqualPv = true + break + } } - if equal := internal.EqualPersistentVolumes(pv1, pv2); !equal { - return false, nil + if !hasEqualPv { + return false, fmt.Errorf("PV %s not found in second list", pv1.Name) } } return true, nil } -func equalPvcLists(pvcList1, pvcList2 *[]corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (bool, error) { +func equalPvcLists(pvcList1, pvcList2 *[]corev1.PersistentVolumeClaim) (bool, error) { if len(*pvcList1) != len(*pvcList2) { return false, nil } - pvcMap1 := internal.GetPersistentVolumeClaimMap(pvcList1, ignoreRandomPostfixInName) - pvcMap2 := internal.GetPersistentVolumeClaimMap(pvcList2, ignoreRandomPostfixInName) - for pvcName, pvc1 := range pvcMap1 { - pvc2, ok := pvcMap2[pvcName] - if !ok { - return false, fmt.Errorf("PVS not found by name %s in second list", pvcName) + for _, pvc1 := range *pvcList1 { + var hasEqualPvc bool + for _, pvc2 := range *pvcList2 { + if !internal.EqualPersistentVolumeClaims(&pvc1, &pvc2) { + continue + } + s := pvc1.Spec.VolumeName[:20] + if s != pvc2.Spec.VolumeName[:20] { + continue + } + s2 := pvc1.GetName()[:len(pvc1.GetName())-5] + if s2 != pvc2.Spec.VolumeName[:len(pvc2.GetName())-5] { + continue + } + hasEqualPvc = true + break } - if equal, err := equalPvcs(pvc1, pvc2, ignoreRandomPostfixInName); err != nil || !equal { - return false, err + if !hasEqualPvc { + return false, fmt.Errorf("PVC %s not found in second list", pvc1.Name) } } return true, nil diff --git a/pkg/apis/internal/persistentvolume.go b/pkg/apis/internal/persistentvolume.go index 104e6455c..1bde8ab41 100644 --- a/pkg/apis/internal/persistentvolume.go +++ b/pkg/apis/internal/persistentvolume.go @@ -8,18 +8,11 @@ import ( corev1 "k8s.io/api/core/v1" ) -// GetPersistentVolumeMap Get map from PersistentVolumeList with name as key -func GetPersistentVolumeMap(pvList *[]corev1.PersistentVolume) map[string]*corev1.PersistentVolume { - pvMap := make(map[string]*corev1.PersistentVolume) - for _, pv := range *pvList { - pv := pv - pvMap[pv.Name] = &pv - } - return pvMap -} - // EqualPersistentVolumes Compare two PersistentVolumes func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { + if pv1 == nil || pv2 == nil { + return false + } // Ignore for now, due to during transition period this would affect existing volume mounts, managed by a provisioner // if !utils.EqualStringMaps(pv1.GetLabels(), pv2.GetLabels()) { // return false, nil @@ -57,6 +50,88 @@ func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { return true } +// EqualPersistentVolumesForTest Compare two PersistentVolumes for test +func EqualPersistentVolumesForTest(pv1, pv2 *corev1.PersistentVolume) bool { + if pv1 == nil || pv2 == nil { + return false + } + // Ignore for now, due to during transition period this would affect existing volume mounts, managed by a provisioner + // if !utils.EqualStringMaps(pv1.GetLabels(), pv2.GetLabels()) { + // return false, nil + // } + if !utils.EqualStringMaps(getPvAnnotations(pv1), getPvAnnotations(pv2)) { + return false + } + const ( + pvNameAttrKey = "csi.storage.k8s.io/pv/name" + pvcNameAttrKey = "csi.storage.k8s.io/pvc/name" + ) + pv1Attrs := cloneMap(pv1.Spec.CSI.VolumeAttributes, pvNameAttrKey, pvcNameAttrKey) + pv2Attrs := cloneMap(pv2.Spec.CSI.VolumeAttributes, pvNameAttrKey, pvcNameAttrKey) + if !utils.EqualStringMaps(pv1Attrs, pv2Attrs) { + return false + } + pv1NameAttr := pv1.Spec.CSI.VolumeAttributes[pvNameAttrKey] + pv2NameAttr := pv2.Spec.CSI.VolumeAttributes[pvNameAttrKey] + if len(pv1NameAttr) == 0 || len(pv2NameAttr) == 0 { + return false + } + s := pv1NameAttr[:20] + if s != pv2NameAttr[:20] { + return false + } + pvc1NameAttr := pv1.Spec.CSI.VolumeAttributes[pvcNameAttrKey] + pvc2NameAttr := pv2.Spec.CSI.VolumeAttributes[pvcNameAttrKey] + s2 := pvc1NameAttr[:len(pvc1NameAttr)-5] + if s2 != pvc2NameAttr[:len(pvc1NameAttr)-5] { + return false + } + + if !utils.EqualStringMaps(getMountOptionsMap(pv1.Spec.MountOptions), getMountOptionsMap(pv2.Spec.MountOptions)) { + return false + } + // ignore pv1.Spec.StorageClassName != pv2.Spec.StorageClassName for transition period + if pv1.Spec.Capacity[corev1.ResourceStorage] != pv2.Spec.Capacity[corev1.ResourceStorage] || + len(pv1.Spec.AccessModes) != len(pv2.Spec.AccessModes) || + (len(pv1.Spec.AccessModes) != 1 && pv1.Spec.AccessModes[0] != pv2.Spec.AccessModes[0]) || + pv1.Spec.CSI.Driver != pv2.Spec.CSI.Driver { + return false + } + if pv1.Spec.CSI.NodeStageSecretRef != nil { + if pv2.Spec.CSI.NodeStageSecretRef == nil || pv1.Spec.CSI.NodeStageSecretRef.Name != pv2.Spec.CSI.NodeStageSecretRef.Name { + return false + } + } else if pv2.Spec.CSI.NodeStageSecretRef != nil { + return false + } + if pv1.Spec.ClaimRef != nil { + if pv2.Spec.ClaimRef == nil { // ignore || pv1.Spec.ClaimRef.Name != pv2.Spec.ClaimRef.Name + return false + } + } else if pv2.Spec.ClaimRef != nil { + return false + } + return true +} + +func cloneMap(original map[string]string, ignoreKeys ...string) map[string]string { + copy := make(map[string]string, len(original)) + ignoreKeysMap := convertToSet(ignoreKeys) + for key, value := range original { + if _, ok := ignoreKeysMap[key]; !ok { + copy[key] = value + } + } + return copy +} + +func convertToSet(ignoreKeys []string) map[string]struct{} { + return slice.Reduce(ignoreKeys, make(map[string]struct{}), func(acc map[string]struct{}, item string) map[string]struct{} { + acc[item] = struct{}{} + return acc + }) +} + func getPvAnnotations(pv *corev1.PersistentVolume) map[string]string { annotations := make(map[string]string) for key, value := range pv.GetAnnotations() { diff --git a/pkg/apis/internal/persistentvolumeclaim.go b/pkg/apis/internal/persistentvolumeclaim.go index 5284a1afb..1564a8087 100644 --- a/pkg/apis/internal/persistentvolumeclaim.go +++ b/pkg/apis/internal/persistentvolumeclaim.go @@ -6,14 +6,11 @@ import ( ) // GetPersistentVolumeClaimMap Get map from PersistentVolumeClaim with name as key -func GetPersistentVolumeClaimMap(pvcList *[]corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) map[string]*corev1.PersistentVolumeClaim { +func GetPersistentVolumeClaimMap(pvcList *[]corev1.PersistentVolumeClaim) map[string]*corev1.PersistentVolumeClaim { pvcMap := make(map[string]*corev1.PersistentVolumeClaim) for _, pvc := range *pvcList { pvc := pvc name := pvc.Name - if ignoreRandomPostfixInName { - name = utils.ShortenString(name, 6) - } pvcMap[name] = &pvc } return pvcMap @@ -21,6 +18,9 @@ func GetPersistentVolumeClaimMap(pvcList *[]corev1.PersistentVolumeClaim, ignore // EqualPersistentVolumeClaims Compare two PersistentVolumeClaims func EqualPersistentVolumeClaims(pvc1, pvc2 *corev1.PersistentVolumeClaim) bool { + if pvc1 == nil || pvc2 == nil { + return false + } if pvc1.GetNamespace() != pvc2.GetNamespace() { return false } From 39d48ed2284281bbed7f4b3bad792975149243a3 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 6 Dec 2024 10:20:59 +0100 Subject: [PATCH 13/68] Correcting unit-tests --- pkg/apis/deployment/volumemount_test.go | 68 ++++++++++--------------- 1 file changed, 27 insertions(+), 41 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index d78a51fa3..993edab34 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -884,14 +884,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), &deployComponent, desiredDeployment) assert.Nil(t, err) - existingPvcs, existingScs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment, utils.GetEnvironmentNamespace(appName, environment)) + existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment, utils.GetEnvironmentNamespace(appName, environment)) assert.Nil(t, err) - equalPvcLists, err := equalPvcLists(&scenario.existingPvcsAfterTestRun, &existingPvcs) - assert.Nil(t, err) - assert.True(t, equalPvcLists) - equalPersistentVolumeLists, err := equalPersistentVolumeLists(&scenario.existingPersistentVolumeAfterTestRun, &existingScs) - assert.Nil(t, err) - assert.True(t, equalPersistentVolumeLists) + assert.True(t, equalPersistentVolumeClaims(&scenario.existingPvcsAfterTestRun, &existingPvcs), "PVC-s are not equal") + assert.True(t, equalPersistentVolumes(&scenario.existingPersistentVolumeAfterTestRun, &existingPvs), "PV-s are not equal") } } }) @@ -1296,6 +1292,7 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co "-o attr_timeout=120", "-o entry_timeout=120", "-o negative_timeout=120", + "-o ro", } idOption := getPersistentVolumeIdMountOption(props) if len(idOption) > 0 { @@ -1331,6 +1328,10 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co csiVolumeMountAttributeSecretNamespace: props.namespace, // skip auto-created by the provisioner "storage.kubernetes.io/csiProvisionerIdentity": "1732528668611-2190-blob.csi.azure.com" }, + NodeStageSecretRef: &corev1.SecretReference{ + Name: props.scSecretName, + Namespace: props.namespace, + }, }, }, AccessModes: []corev1.PersistentVolumeAccessMode{props.volumeAccessMode}, @@ -1441,9 +1442,9 @@ func createBlobFuse2RadixVolumeMount(props expectedPvcScProperties, modify func( return volumeMount } -func equalPersistentVolumeLists(list1, list2 *[]corev1.PersistentVolume) (bool, error) { +func equalPersistentVolumes(list1, list2 *[]corev1.PersistentVolume) bool { if len(*list1) != len(*list2) { - return false, fmt.Errorf("different PersistentVolume list sizes: %v, %v", len(*list1), len(*list2)) + return false } for _, pv1 := range *list1 { var hasEqualPv bool @@ -1454,54 +1455,39 @@ func equalPersistentVolumeLists(list1, list2 *[]corev1.PersistentVolume) (bool, } } if !hasEqualPv { - return false, fmt.Errorf("PV %s not found in second list", pv1.Name) + return false } } - return true, nil + return true } -func equalPvcLists(pvcList1, pvcList2 *[]corev1.PersistentVolumeClaim) (bool, error) { +func equalPersistentVolumeClaims(pvcList1, pvcList2 *[]corev1.PersistentVolumeClaim) bool { if len(*pvcList1) != len(*pvcList2) { - return false, nil + return false } for _, pvc1 := range *pvcList1 { var hasEqualPvc bool for _, pvc2 := range *pvcList2 { - if !internal.EqualPersistentVolumeClaims(&pvc1, &pvc2) { - continue - } - s := pvc1.Spec.VolumeName[:20] - if s != pvc2.Spec.VolumeName[:20] { - continue - } - s2 := pvc1.GetName()[:len(pvc1.GetName())-5] - if s2 != pvc2.Spec.VolumeName[:len(pvc2.GetName())-5] { - continue + if equalNamePrefix(pvc1, pvc2) && + equalVolumeNamePrefix(pvc1, pvc2) && + internal.EqualPersistentVolumeClaims(&pvc1, &pvc2) { + hasEqualPvc = true + break } - hasEqualPvc = true - break } if !hasEqualPvc { - return false, fmt.Errorf("PVC %s not found in second list", pvc1.Name) + return false } } - return true, nil + return true } -func equalPvcs(pvc1 *corev1.PersistentVolumeClaim, pvc2 *corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (bool, error) { - pvc1Copy, labels1 := getPvcCopyWithLabels(pvc1, ignoreRandomPostfixInName) - pvc2Copy, labels2 := getPvcCopyWithLabels(pvc2, ignoreRandomPostfixInName) - patchBytes, err := getPvcPatch(pvc1Copy, pvc2Copy) - if err != nil { - return false, err - } - if !utils.EqualStringMaps(labels1, labels2) { - return false, fmt.Errorf("PVC-s labels are not equal") - } - if !kube.IsEmptyPatch(patchBytes) { - return false, fmt.Errorf("PVC-s are not equal: %s", patchBytes) - } - return true, nil +func equalVolumeNamePrefix(pvc1 corev1.PersistentVolumeClaim, pvc2 corev1.PersistentVolumeClaim) bool { + return pvc1.Spec.VolumeName[:20] == pvc2.Spec.VolumeName[:20] +} + +func equalNamePrefix(pvc1 corev1.PersistentVolumeClaim, pvc2 corev1.PersistentVolumeClaim) bool { + return pvc1.GetName()[:len(pvc1.GetName())-5] == pvc2.GetName()[:len(pvc2.GetName())-5] } func getPvcCopyWithLabels(pvc *corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (*corev1.PersistentVolumeClaim, map[string]string) { From 49390b851c03b592b6646d236b5db2b9b0edbb75 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 6 Dec 2024 15:10:33 +0100 Subject: [PATCH 14/68] Correcting unit-tests --- pkg/apis/deployment/volumemount.go | 110 ++++------ pkg/apis/deployment/volumemount_test.go | 207 +++++++++--------- .../internal/persistentvolume/defaults.go | 36 +++ .../persistentvolume.go | 62 +++--- .../persistentvolumeclaim.go | 2 +- 5 files changed, 206 insertions(+), 211 deletions(-) create mode 100644 pkg/apis/internal/persistentvolume/defaults.go rename pkg/apis/internal/{ => persistentvolume}/persistentvolume.go (61%) rename pkg/apis/internal/{ => persistentvolume}/persistentvolumeclaim.go (98%) diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index 937489017..db843e4c1 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -12,7 +12,7 @@ import ( "github.com/equinor/radix-common/utils/slice" "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/defaults/k8s" - "github.com/equinor/radix-operator/pkg/apis/internal" + "github.com/equinor/radix-operator/pkg/apis/internal/persistentvolume" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/google/uuid" @@ -26,44 +26,10 @@ import ( ) const ( - csiVolumeTypeBlobFuse2ProtocolFuse = "csi-az-blob" - csiVolumeTypeBlobFuse2ProtocolFuse2 = "csi-blobfuse2-fuse2" - csiVolumeNameTemplate = "%s-%s-%s-%s" // --- - csiPersistentVolumeClaimNameTemplate = "pvc-%s-%s" // pvc-- - csiPersistentVolumeNameTemplate = "pv-radixvolumemount-%s" // pv- - - csiPersistentVolumeProvisionerSecretNameParameter = "csi.storage.k8s.io/provisioner-secret-name" // Secret name, containing storage account name and key - csiPersistentVolumeProvisionerSecretNamespaceParameter = "csi.storage.k8s.io/provisioner-secret-namespace" // namespace of the secret - csiPersistentVolumeNodeStageSecretNameParameter = "csi.storage.k8s.io/node-stage-secret-name" // Usually equal to csiPersistentVolumeProvisionerSecretNameParameter - csiPersistentVolumeNodeStageSecretNamespaceParameter = "csi.storage.k8s.io/node-stage-secret-namespace" // Usually equal to csiPersistentVolumeProvisionerSecretNamespaceParameter - csiVolumeMountAttributeContainerName = "containerName" // Container name - foc container storages - csiPersistentVolumeTmpPathMountOption = "tmp-path" // Path within the node, where the volume mount has been mounted to - csiPersistentVolumeGidMountOption = "gid" // Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting - csiPersistentVolumeUidMountOption = "uid" // Volume mount owner UserID. Used instead of GroupID - csiPersistentVolumeUseAdlsMountOption = "use-adls" // Use ADLS or Block Blob - csiPersistentVolumeStreamingEnabledMountOption = "streaming" // Enable Streaming - csiPersistentVolumeStreamingCacheMountOption = "stream-cache-mb" // Limit total amount of data being cached in memory to conserve memory - csiPersistentVolumeStreamingMaxBlocksPerFileMountOption = "max-blocks-per-file" // Maximum number of blocks to be cached in memory for streaming - csiPersistentVolumeStreamingMaxBuffersMountOption = "max-buffers" // The total number of buffers to be cached in memory (in MB). - csiPersistentVolumeStreamingBlockSizeMountOption = "block-size-mb" // The size of each block to be cached in memory (in MB). - csiPersistentVolumeStreamingBufferSizeMountOption = "buffer-size-mb" // The size of each buffer to be cached in memory (in MB). - csiVolumeMountAttributeProtocol = "protocol" // Protocol - csiPersistentVolumeProtocolParameterFuse = "fuse" // Protocol "blobfuse" - csiPersistentVolumeProtocolParameterFuse2 = "fuse2" // Protocol "blobfuse2" - csiVolumeMountAttributeStorageAccount = "storageAccount" - csiVolumeMountAttributeClientID = "clientID" - csiVolumeMountAttributeResourceGroup = "resourcegroup" - csiVolumeMountAttributeSecretNamespace = "secretnamespace" - csiVolumeMountAttributePvName = "csi.storage.k8s.io/pv/name" - csiVolumeMountAttributePvcName = "csi.storage.k8s.io/pvc/name" - csiVolumeMountAttributePvcNamespace = "csi.storage.k8s.io/pvc/namespace" - csiVolumeMountAnnotationProvisionedBy = "pv.kubernetes.io/provisioned-by" - csiVolumeMountAnnotationProvisionerDeletionSecretName = "volume.kubernetes.io/provisioner-deletion-secret-name" - csiVolumeMountAnnotationProvisionerDeletionSecretNamespace = "volume.kubernetes.io/provisioner-deletion-secret-namespace" - - csiSecretStoreDriver = "secrets-store.csi.k8s.io" - csiVolumeSourceVolumeAttrSecretProviderClassName = "secretProviderClass" - csiAzureKeyVaultSecretMountPathTemplate = "/mnt/azure-key-vault/%s" + csiVolumeTypeBlobFuse2ProtocolFuse = "csi-az-blob" + csiVolumeTypeBlobFuse2ProtocolFuse2 = "csi-blobfuse2-fuse2" + csiVolumeNameTemplate = "%s-%s-%s-%s" // --- + csiAzureKeyVaultSecretMountPathTemplate = "/mnt/azure-key-vault/%s" volumeNameMaxLength = 63 ) @@ -240,9 +206,9 @@ func getComponentSecretRefsAzureKeyVaultVolumes(ctx context.Context, kubeutil *k switch provider { case "azure": volume.VolumeSource.CSI = &corev1.CSIVolumeSource{ - Driver: csiSecretStoreDriver, + Driver: persistentvolume.CsiVolumeSourceDriverSecretStore, ReadOnly: pointers.Ptr(true), - VolumeAttributes: map[string]string{csiVolumeSourceVolumeAttrSecretProviderClassName: secretProviderClass.Name}, + VolumeAttributes: map[string]string{persistentvolume.CsiVolumeSourceVolumeAttributeSecretProviderClass: secretProviderClass.Name}, } useAzureIdentity := azureKeyVault.UseAzureIdentity != nil && *azureKeyVault.UseAzureIdentity @@ -391,11 +357,11 @@ func getCsiAzurePersistentVolumeClaimName(componentName string, radixVolumeMount if err != nil { return "", err } - return fmt.Sprintf(csiPersistentVolumeClaimNameTemplate, volumeName, strings.ToLower(commonUtils.RandString(5))), nil + return fmt.Sprintf(persistentvolume.CsiPersistentVolumeClaimNameTemplate, volumeName, strings.ToLower(commonUtils.RandString(5))), nil } func getCsiAzurePersistentVolumeName() string { - return fmt.Sprintf(csiPersistentVolumeNameTemplate, uuid.New().String()) + return fmt.Sprintf(persistentvolume.CsiPersistentVolumeNameTemplate, uuid.New().String()) } func (deploy *Deployment) createOrUpdateCsiAzureVolumeMountsSecrets(ctx context.Context, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, accountName, accountKey []byte) error { @@ -530,7 +496,7 @@ func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, persistentVolume.Spec.CSI = &corev1.CSIPersistentVolumeSource{ Driver: provisionerBlobCsiAzure, VolumeHandle: getVolumeHandle(namespace, componentName, pvName, getRadixVolumeMountStorage(radixVolumeMount)), - VolumeAttributes: getCsiAzurePersistentVolumeAttributes(namespace, radixVolumeMount, pvName, pvcName, useAzureIdentity, identityClientId), + VolumeAttributes: getCsiAzurePersistentVolumeAttributes(namespace, radixVolumeMount, pvName, pvcName, useAzureIdentity, identityClientId, csiVolumeCredSecretName), } if !useAzureIdentity { persistentVolume.Spec.CSI.NodeStageSecretRef = &corev1.SecretReference{Name: csiVolumeCredSecretName, Namespace: namespace} @@ -558,11 +524,11 @@ func getIdentityClientId(identity *radixv1.Identity) string { func getCsiAzurePersistentVolumeAnnotations(csiVolumeCredSecretName, namespace string, useAzureIdentity bool) map[string]string { annotationsMap := map[string]string{ - csiVolumeMountAnnotationProvisionedBy: provisionerBlobCsiAzure, + persistentvolume.CsiAnnotationProvisionedBy: provisionerBlobCsiAzure, } if !useAzureIdentity { - annotationsMap[csiVolumeMountAnnotationProvisionerDeletionSecretName] = csiVolumeCredSecretName - annotationsMap[csiVolumeMountAnnotationProvisionerDeletionSecretNamespace] = namespace + annotationsMap[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = csiVolumeCredSecretName + annotationsMap[persistentvolume.CsiAnnotationProvisionerDeletionSecretNamespace] = namespace } return annotationsMap } @@ -576,28 +542,28 @@ func getCsiAzurePersistentVolumeLabels(appName, namespace, componentName string, } } -func getCsiAzurePersistentVolumeAttributes(namespace string, radixVolumeMount *radixv1.RadixVolumeMount, pvName, pvcName string, useAzureIdentity bool, clientId string) map[string]string { +func getCsiAzurePersistentVolumeAttributes(namespace string, radixVolumeMount *radixv1.RadixVolumeMount, pvName, pvcName string, useAzureIdentity bool, clientId, csiVolumeCredSecretName string) map[string]string { attributes := make(map[string]string) switch GetCsiAzureVolumeMountType(radixVolumeMount) { case radixv1.MountTypeBlobFuse2FuseCsiAzure: - attributes[csiVolumeMountAttributeContainerName] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) - attributes[csiVolumeMountAttributeProtocol] = csiPersistentVolumeProtocolParameterFuse + attributes[persistentvolume.CsiVolumeMountAttributeContainerName] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) + attributes[persistentvolume.CsiVolumeMountAttributeProtocol] = persistentvolume.CsiVolumeAttributeProtocolParameterFuse case radixv1.MountTypeBlobFuse2Fuse2CsiAzure: - attributes[csiVolumeMountAttributeContainerName] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) - attributes[csiVolumeMountAttributeProtocol] = csiPersistentVolumeProtocolParameterFuse2 + attributes[persistentvolume.CsiVolumeMountAttributeContainerName] = getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount) + attributes[persistentvolume.CsiVolumeMountAttributeProtocol] = persistentvolume.CsiVolumeAttributeProtocolParameterFuse2 if len(radixVolumeMount.BlobFuse2.StorageAccount) > 0 { - attributes[csiVolumeMountAttributeStorageAccount] = radixVolumeMount.BlobFuse2.StorageAccount + attributes[persistentvolume.CsiVolumeAttributeStorageAccount] = radixVolumeMount.BlobFuse2.StorageAccount } if useAzureIdentity { - attributes[csiVolumeMountAttributeClientID] = clientId - attributes[csiVolumeMountAttributeResourceGroup] = radixVolumeMount.BlobFuse2.ResourceGroup + attributes[persistentvolume.CsiVolumeAttributeClientID] = clientId + attributes[persistentvolume.CsiVolumeAttributeResourceGroup] = radixVolumeMount.BlobFuse2.ResourceGroup } } - attributes[csiVolumeMountAttributePvName] = pvName - attributes[csiVolumeMountAttributePvcName] = pvcName - attributes[csiVolumeMountAttributePvcNamespace] = namespace + attributes[persistentvolume.CsiVolumeMountAttributePvName] = pvName + attributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvcName + attributes[persistentvolume.CsiVolumeMountAttributePvcNamespace] = namespace if !useAzureIdentity { - attributes[csiVolumeMountAttributeSecretNamespace] = namespace + attributes[persistentvolume.CsiVolumeMountAttributeSecretNamespace] = namespace } // Do not specify the key storage.kubernetes.io/csiProvisionerIdentity in csi.volumeAttributes in PV specification. This key indicates dynamically provisioned PVs // It looks like: storage.kubernetes.io/csiProvisionerIdentity: 1731647415428-2825-blob.csi.azure.com @@ -616,11 +582,11 @@ func getCsiAzurePersistentVolumeMountOptionsForAzureBlob(radixVolumeMount *radix } gid := getRadixBlobFuse2VolumeMountGid(radixVolumeMount) if len(gid) > 0 { - mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", csiPersistentVolumeGidMountOption, gid)) + mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", persistentvolume.CsiMountOptionGid, gid)) } else { uid := getRadixBlobFuse2VolumeMountUid(radixVolumeMount) if len(uid) > 0 { - mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", csiPersistentVolumeUidMountOption, uid)) + mountOptions = append(mountOptions, fmt.Sprintf("-o %s=%s", persistentvolume.CsiMountOptionUid, uid)) } } if getVolumeMountAccessMode(radixVolumeMount) == corev1.ReadOnlyMany { @@ -628,7 +594,7 @@ func getCsiAzurePersistentVolumeMountOptionsForAzureBlob(radixVolumeMount *radix } if radixVolumeMount.BlobFuse2 != nil { mountOptions = append(mountOptions, getStreamingMountOptions(radixVolumeMount.BlobFuse2.Streaming)...) - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeUseAdlsMountOption, radixVolumeMount.BlobFuse2.UseAdls != nil && *radixVolumeMount.BlobFuse2.UseAdls)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", persistentvolume.CsiMountOptionUseAdls, radixVolumeMount.BlobFuse2.UseAdls != nil && *radixVolumeMount.BlobFuse2.UseAdls)) } return mountOptions } @@ -638,24 +604,24 @@ func getStreamingMountOptions(streaming *radixv1.RadixVolumeMountStreaming) []st if streaming != nil && streaming.Enabled != nil && !*streaming.Enabled { return nil } - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%t", csiPersistentVolumeStreamingEnabledMountOption, true)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%t", persistentvolume.CsiMountOptionStreamingEnabled, true)) if streaming == nil { return mountOptions } if streaming.StreamCache != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeStreamingCacheMountOption, *streaming.StreamCache)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", persistentvolume.CsiMountOptionStreamingCache, *streaming.StreamCache)) } if streaming.BlockSize != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeStreamingBlockSizeMountOption, *streaming.BlockSize)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", persistentvolume.CsiMountOptionStreamingBlockSize, *streaming.BlockSize)) } if streaming.BufferSize != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeStreamingBufferSizeMountOption, *streaming.BufferSize)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", persistentvolume.CsiMountOptionStreamingBufferSize, *streaming.BufferSize)) } if streaming.MaxBuffers != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeStreamingMaxBuffersMountOption, *streaming.MaxBuffers)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", persistentvolume.CsiMountOptionStreamingMaxBuffers, *streaming.MaxBuffers)) } if streaming.MaxBlocksPerFile != nil { - mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", csiPersistentVolumeStreamingMaxBlocksPerFileMountOption, *streaming.MaxBlocksPerFile)) + mountOptions = append(mountOptions, fmt.Sprintf("--%s=%v", persistentvolume.CsiMountOptionStreamingMaxBlocksPerFile, *streaming.MaxBlocksPerFile)) } return mountOptions } @@ -849,7 +815,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolume(ctx con if err != nil { return nil, err } - if pvcExist && !internal.EqualPersistentVolumeClaims(existingPvc, newPvc) { + if pvcExist && !persistentvolume.EqualPersistentVolumeClaims(existingPvc, newPvc) { pvcName = newPvc.GetName() } if !actualPvExists { @@ -859,7 +825,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolume(ctx con return nil, err } } - if !pvcExist || !internal.EqualPersistentVolumeClaims(existingPvc, newPvc) { + if !pvcExist || !persistentvolume.EqualPersistentVolumeClaims(existingPvc, newPvc) { newPvc.SetName(pvcName) log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for PersistentVolume %s", newPvc.GetName(), namespace, pvName) if _, err := deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, newPvc, metav1.CreateOptions{}); err != nil { @@ -875,7 +841,7 @@ func (deploy *Deployment) getPvcByNameMap(ctx context.Context, namespace string, if err != nil { return nil, err } - return internal.GetPersistentVolumeClaimMap(&pvcList.Items), nil + return persistentvolume.GetPersistentVolumeClaimMap(&pvcList.Items), nil } func (deploy *Deployment) getCurrentlyUsedPersistentVolumeClaims(ctx context.Context, desiredDeployment *appsv1.Deployment) (map[string]any, error) { @@ -939,7 +905,7 @@ func getActualExistingCsiAzurePvNameByPvcName(appName, namespace, componentName for _, existingPv := range existingPvList { if existingPv.Spec.ClaimRef != nil && existingPv.Spec.ClaimRef.Name == pvcName { desiredPv := populateCsiAzurePersistentVolume(existingPv.DeepCopy(), appName, namespace, componentName, existingPv.GetName(), pvcName, radixVolumeMount, identity) - if internal.EqualPersistentVolumes(&existingPv, desiredPv) { + if persistentvolume.EqualPersistentVolumes(&existingPv, desiredPv) { return existingPv.GetName(), true } } diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 993edab34..df4f76f8b 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -10,7 +10,7 @@ import ( commonUtils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/config" - "github.com/equinor/radix-operator/pkg/apis/internal" + "github.com/equinor/radix-operator/pkg/apis/internal/persistentvolume" "github.com/equinor/radix-operator/pkg/apis/kube" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" @@ -507,93 +507,90 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { componentName := "some-component" var scenarios []deploymentVolumesTestScenario - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, - existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // type scenarioProperties struct { - // changedNewRadixVolumeName string - // changedNewRadixVolumeStorageName string - // expectedVolumeName string - // expectedNewSecretName string - // expectedNewPvcName string - // expectedNewPvName string - // expectedNewScTmpPath string - // } - // getScenario := func(props expectedPvcScProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ - // name: "Update storage in existing volume name and storage", + // name: "Create new volume", // props: props, // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.Name = scenarioProps.changedNewRadixVolumeName - // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - // }), + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), // }, // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) { - // v.Name = scenarioProps.expectedVolumeName - // }), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // createTestVolume(props, func(v *corev1.Volume) {}), // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - // }), - // }, - // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), // }, + // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) { - // sc.ObjectMeta.Name = scenarioProps.expectedNewPvName - // sc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // // setPersistentVolumeMountOption(sc, "--tmp-path", scenarioProps.expectedNewScTmpPath) //TODO: this option does not work with blobfuse2 in some reason - investigate to make use separate disk volume for csi volumes - // setVolumeMountAttribute(props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, sc) - // sc.Spec.CSI.VolumeAttributes[csiPersistentVolumeProvisionerSecretNameParameter] = scenarioProps.expectedNewSecretName - // sc.Spec.CSI.VolumeAttributes[csiPersistentVolumeNodeStageSecretNameParameter] = scenarioProps.expectedNewSecretName - // }), + // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), // }, // } // } // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - // changedNewRadixVolumeName: "volume101", - // changedNewRadixVolumeStorageName: "storage101", - // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - // expectedNewSecretName: "some-component-volume101-csiazurecreds", - // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - // expectedNewPvName: "pv-radixvolumemount-some-uuid", - // expectedNewScTmpPath: "/tmp/any-app-some-env/csi-az-blob/some-component/volume101/storage101", - // }), + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), // } // }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + type scenarioProperties struct { + changedNewRadixVolumeName string + changedNewRadixVolumeStorageName string + expectedVolumeName string + expectedNewSecretName string + expectedNewPvcName string + expectedNewPvName string + } + getScenario := func(props expectedPvcScProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Update storage in existing volume name and storage", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.Name = scenarioProps.changedNewRadixVolumeName + vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) { + v.Name = scenarioProps.expectedVolumeName + }), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + }), + }, + existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), + }, + existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName + setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + changedNewRadixVolumeName: "volume101", + changedNewRadixVolumeStorageName: "storage101", + expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + expectedNewSecretName: "some-component-volume101-csiazurecreds", + expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + expectedNewPvName: "pv-radixvolumemount-some-uuid", + }), + } + }()...) // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { // persistentVolumeForAnotherNamespace := createRandomPersistentVolume(props, utils.RandString(10), utils.RandString(10)) @@ -625,7 +622,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // persistentVolumeForAnotherComponent, // }, // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), + // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), // persistentVolumeForAnotherNamespace, // persistentVolumeForAnotherComponent, // }, @@ -691,7 +688,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // createRandomPersistentVolume(props, props.namespace, props.componentName), // }, // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), + // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), // }, // } // } @@ -722,7 +719,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // createRandomPersistentVolume(props, props.namespace, props.componentName), // }, // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) {}), + // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), // }, // } // } @@ -1107,10 +1104,10 @@ func Test_EmptyDir(t *testing.T) { } func createRandomPersistentVolume(props expectedPvcScProperties, namespace, componentName string) corev1.PersistentVolume { - return createExpectedVolumeMount(props, func(sc *corev1.PersistentVolume) { - sc.ObjectMeta.Name = utils.RandString(10) - sc.ObjectMeta.Labels[kube.RadixNamespace] = namespace - sc.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName + return createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + pv.ObjectMeta.Name = utils.RandString(10) + pv.ObjectMeta.Labels[kube.RadixNamespace] = namespace + pv.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName }) } @@ -1124,8 +1121,8 @@ func createRandomPvc(props expectedPvcScProperties, namespace, componentName str } // TODO: this option does not work with blobfuse2 in some reason - investigate to make use separate disk volume for csi volumes -// func setPersistentVolumeMountOption(sc *corev1.PersistentVolume, key, value string) { -// mountOptions := sc.MountOptions +// func setPersistentVolumeMountOption(pv *corev1.PersistentVolume, key, value string) { +// mountOptions := pv.MountOptions // for i, option := range mountOptions { // if strings.Contains(option, key) { // mountOptions[i] = fmt.Sprintf("%s=%s", key, value) @@ -1298,7 +1295,7 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co if len(idOption) > 0 { mountOptions = append(mountOptions, idOption) } - sc := corev1.PersistentVolume{ + pv := corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: props.persistentVolumeName, Labels: map[string]string{ @@ -1308,9 +1305,9 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co kube.RadixVolumeMountNameLabel: props.radixVolumeMountName, }, Annotations: map[string]string{ - csiVolumeMountAnnotationProvisionedBy: provisionerBlobCsiAzure, - csiVolumeMountAnnotationProvisionerDeletionSecretName: props.scSecretName, - csiVolumeMountAnnotationProvisionerDeletionSecretNamespace: props.namespace, + persistentvolume.CsiAnnotationProvisionedBy: provisionerBlobCsiAzure, + persistentvolume.CsiAnnotationProvisionerDeletionSecretName: props.scSecretName, + persistentvolume.CsiAnnotationProvisionerDeletionSecretNamespace: props.namespace, }, }, Spec: corev1.PersistentVolumeSpec{ @@ -1320,12 +1317,12 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co Driver: provisionerBlobCsiAzure, VolumeHandle: getVolumeHandle(props.namespace, props.componentName, props.persistentVolumeName, props.radixStorageName), VolumeAttributes: map[string]string{ - csiVolumeMountAttributeContainerName: props.radixStorageName, - csiVolumeMountAttributeProtocol: csiPersistentVolumeProtocolParameterFuse2, - csiVolumeMountAttributePvName: props.persistentVolumeName, - csiVolumeMountAttributePvcName: props.pvcName, - csiVolumeMountAttributePvcNamespace: props.namespace, - csiVolumeMountAttributeSecretNamespace: props.namespace, + persistentvolume.CsiVolumeMountAttributeContainerName: props.radixStorageName, + persistentvolume.CsiVolumeMountAttributeProtocol: persistentvolume.CsiVolumeAttributeProtocolParameterFuse2, + persistentvolume.CsiVolumeMountAttributePvName: props.persistentVolumeName, + persistentvolume.CsiVolumeMountAttributePvcName: props.pvcName, + persistentvolume.CsiVolumeMountAttributePvcNamespace: props.namespace, + persistentvolume.CsiVolumeMountAttributeSecretNamespace: props.namespace, // skip auto-created by the provisioner "storage.kubernetes.io/csiProvisionerIdentity": "1732528668611-2190-blob.csi.azure.com" }, NodeStageSecretRef: &corev1.SecretReference{ @@ -1346,21 +1343,21 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, }, } - setVolumeMountAttribute(props.radixVolumeMountType, props.radixStorageName, &sc) + setVolumeMountAttribute(&pv, props.radixVolumeMountType, props.radixStorageName, props.pvcName) if modify != nil { - modify(&sc) + modify(&pv) } - return sc + return pv } -func setVolumeMountAttribute(radixVolumeMountType v1.MountType, containerName string, pv *corev1.PersistentVolume) { +func setVolumeMountAttribute(pv *corev1.PersistentVolume, radixVolumeMountType v1.MountType, containerName, pvcName string) { + pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributeContainerName] = containerName + pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvcName switch radixVolumeMountType { case v1.MountTypeBlobFuse2FuseCsiAzure: - pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeContainerName] = containerName - pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeProtocol] = csiPersistentVolumeProtocolParameterFuse + pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributeProtocol] = persistentvolume.CsiVolumeAttributeProtocolParameterFuse case v1.MountTypeBlobFuse2Fuse2CsiAzure: - pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeContainerName] = containerName - pv.Spec.CSI.VolumeAttributes[csiVolumeMountAttributeProtocol] = csiPersistentVolumeProtocolParameterFuse2 + pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributeProtocol] = persistentvolume.CsiVolumeAttributeProtocolParameterFuse2 } } @@ -1442,14 +1439,14 @@ func createBlobFuse2RadixVolumeMount(props expectedPvcScProperties, modify func( return volumeMount } -func equalPersistentVolumes(list1, list2 *[]corev1.PersistentVolume) bool { - if len(*list1) != len(*list2) { +func equalPersistentVolumes(expectedPvs, actualPvs *[]corev1.PersistentVolume) bool { + if len(*expectedPvs) != len(*actualPvs) { return false } - for _, pv1 := range *list1 { + for _, expectedPv := range *expectedPvs { var hasEqualPv bool - for _, pv2 := range *list2 { - if internal.EqualPersistentVolumesForTest(&pv1, &pv2) { + for _, actualPv := range *actualPvs { + if persistentvolume.EqualPersistentVolumesForTest(&expectedPv, &actualPv) { hasEqualPv = true break } @@ -1470,7 +1467,7 @@ func equalPersistentVolumeClaims(pvcList1, pvcList2 *[]corev1.PersistentVolumeCl for _, pvc2 := range *pvcList2 { if equalNamePrefix(pvc1, pvc2) && equalVolumeNamePrefix(pvc1, pvc2) && - internal.EqualPersistentVolumeClaims(&pvc1, &pvc2) { + persistentvolume.EqualPersistentVolumeClaims(&pvc1, &pvc2) { hasEqualPvc = true break } diff --git a/pkg/apis/internal/persistentvolume/defaults.go b/pkg/apis/internal/persistentvolume/defaults.go new file mode 100644 index 000000000..c65849553 --- /dev/null +++ b/pkg/apis/internal/persistentvolume/defaults.go @@ -0,0 +1,36 @@ +package persistentvolume + +const ( + CsiPersistentVolumeClaimNameTemplate = "pvc-%s-%s" // pvc-- + CsiPersistentVolumeNameTemplate = "pv-radixvolumemount-%s" // pv- + + CsiMountOptionGid = "gid" // Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting + CsiMountOptionUid = "uid" // Volume mount owner UserID. Used instead of GroupID + CsiMountOptionUseAdls = "use-adls" // Use ADLS or Block Blob + CsiMountOptionStreamingEnabled = "streaming" // Enable Streaming + CsiMountOptionStreamingCache = "stream-cache-mb" // Limit total amount of data being cached in memory to conserve memory + CsiMountOptionStreamingMaxBlocksPerFile = "max-blocks-per-file" // Maximum number of blocks to be cached in memory for streaming + CsiMountOptionStreamingMaxBuffers = "max-buffers" // The total number of buffers to be cached in memory (in MB). + CsiMountOptionStreamingBlockSize = "block-size-mb" // The size of each block to be cached in memory (in MB). + CsiMountOptionStreamingBufferSize = "buffer-size-mb" // The size of each buffer to be cached in memory (in MB). + + CsiVolumeAttributeProtocolParameterFuse = "fuse" // Protocol "blobfuse" + CsiVolumeAttributeProtocolParameterFuse2 = "fuse2" // Protocol "blobfuse2" + CsiVolumeAttributeStorageAccount = "storageAccount" + CsiVolumeAttributeClientID = "clientID" + CsiVolumeAttributeResourceGroup = "resourcegroup" + + CsiAnnotationProvisionedBy = "pv.kubernetes.io/provisioned-by" + CsiAnnotationProvisionerDeletionSecretName = "volume.kubernetes.io/provisioner-deletion-secret-name" + CsiAnnotationProvisionerDeletionSecretNamespace = "volume.kubernetes.io/provisioner-deletion-secret-namespace" + + CsiVolumeSourceDriverSecretStore = "secrets-store.csi.k8s.io" + CsiVolumeSourceVolumeAttributeSecretProviderClass = "secretProviderClass" + + CsiVolumeMountAttributePvName = "csi.storage.k8s.io/pv/name" + CsiVolumeMountAttributePvcName = "csi.storage.k8s.io/pvc/name" + CsiVolumeMountAttributePvcNamespace = "csi.storage.k8s.io/pvc/namespace" + CsiVolumeMountAttributeSecretNamespace = "secretnamespace" + CsiVolumeMountAttributeProtocol = "protocol" // Protocol + CsiVolumeMountAttributeContainerName = "containerName" // Container name - foc container storages +) diff --git a/pkg/apis/internal/persistentvolume.go b/pkg/apis/internal/persistentvolume/persistentvolume.go similarity index 61% rename from pkg/apis/internal/persistentvolume.go rename to pkg/apis/internal/persistentvolume/persistentvolume.go index 1bde8ab41..eff16a17a 100644 --- a/pkg/apis/internal/persistentvolume.go +++ b/pkg/apis/internal/persistentvolume/persistentvolume.go @@ -1,4 +1,4 @@ -package internal +package persistentvolume import ( "strings" @@ -51,64 +51,60 @@ func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { } // EqualPersistentVolumesForTest Compare two PersistentVolumes for test -func EqualPersistentVolumesForTest(pv1, pv2 *corev1.PersistentVolume) bool { - if pv1 == nil || pv2 == nil { +func EqualPersistentVolumesForTest(expectedPv, actualPv *corev1.PersistentVolume) bool { + if expectedPv == nil || actualPv == nil { return false } // Ignore for now, due to during transition period this would affect existing volume mounts, managed by a provisioner - // if !utils.EqualStringMaps(pv1.GetLabels(), pv2.GetLabels()) { + // if !utils.EqualStringMaps(expectedPv.GetLabels(), actualPv.GetLabels()) { // return false, nil // } - if !utils.EqualStringMaps(getPvAnnotations(pv1), getPvAnnotations(pv2)) { + if !utils.EqualStringMaps(getPvAnnotations(expectedPv), getPvAnnotations(actualPv)) { return false } - const ( - pvNameAttrKey = "csi.storage.k8s.io/pv/name" - pvcNameAttrKey = "csi.storage.k8s.io/pvc/name" - ) - pv1Attrs := cloneMap(pv1.Spec.CSI.VolumeAttributes, pvNameAttrKey, pvcNameAttrKey) - pv2Attrs := cloneMap(pv2.Spec.CSI.VolumeAttributes, pvNameAttrKey, pvcNameAttrKey) - if !utils.EqualStringMaps(pv1Attrs, pv2Attrs) { + expectedClonedAttrs := cloneMap(expectedPv.Spec.CSI.VolumeAttributes, CsiVolumeMountAttributePvName, CsiVolumeMountAttributePvcName) + actualClonedAttrs := cloneMap(actualPv.Spec.CSI.VolumeAttributes, CsiVolumeMountAttributePvName, CsiVolumeMountAttributePvcName) + if !utils.EqualStringMaps(expectedClonedAttrs, actualClonedAttrs) { return false } - pv1NameAttr := pv1.Spec.CSI.VolumeAttributes[pvNameAttrKey] - pv2NameAttr := pv2.Spec.CSI.VolumeAttributes[pvNameAttrKey] - if len(pv1NameAttr) == 0 || len(pv2NameAttr) == 0 { + expectedNameAttr := expectedPv.Spec.CSI.VolumeAttributes[CsiVolumeMountAttributePvName] + actualNameAttr := actualPv.Spec.CSI.VolumeAttributes[CsiVolumeMountAttributePvName] + if len(expectedNameAttr) == 0 || len(actualNameAttr) == 0 { return false } - s := pv1NameAttr[:20] - if s != pv2NameAttr[:20] { + s := expectedNameAttr[:20] + if s != actualNameAttr[:20] { return false } - pvc1NameAttr := pv1.Spec.CSI.VolumeAttributes[pvcNameAttrKey] - pvc2NameAttr := pv2.Spec.CSI.VolumeAttributes[pvcNameAttrKey] - s2 := pvc1NameAttr[:len(pvc1NameAttr)-5] - if s2 != pvc2NameAttr[:len(pvc1NameAttr)-5] { + expectedPvcNameAttr := expectedPv.Spec.CSI.VolumeAttributes[CsiVolumeMountAttributePvcName] + actualPvcNameAttr := actualPv.Spec.CSI.VolumeAttributes[CsiVolumeMountAttributePvcName] + s2 := expectedPvcNameAttr[:len(expectedPvcNameAttr)-5] + if s2 != actualPvcNameAttr[:len(actualPvcNameAttr)-5] { return false } - if !utils.EqualStringMaps(getMountOptionsMap(pv1.Spec.MountOptions), getMountOptionsMap(pv2.Spec.MountOptions)) { + if !utils.EqualStringMaps(getMountOptionsMap(expectedPv.Spec.MountOptions), getMountOptionsMap(actualPv.Spec.MountOptions)) { return false } - // ignore pv1.Spec.StorageClassName != pv2.Spec.StorageClassName for transition period - if pv1.Spec.Capacity[corev1.ResourceStorage] != pv2.Spec.Capacity[corev1.ResourceStorage] || - len(pv1.Spec.AccessModes) != len(pv2.Spec.AccessModes) || - (len(pv1.Spec.AccessModes) != 1 && pv1.Spec.AccessModes[0] != pv2.Spec.AccessModes[0]) || - pv1.Spec.CSI.Driver != pv2.Spec.CSI.Driver { + // ignore expectedPv.Spec.StorageClassName != actualPv.Spec.StorageClassName for transition period + if expectedPv.Spec.Capacity[corev1.ResourceStorage] != actualPv.Spec.Capacity[corev1.ResourceStorage] || + len(expectedPv.Spec.AccessModes) != len(actualPv.Spec.AccessModes) || + (len(expectedPv.Spec.AccessModes) != 1 && expectedPv.Spec.AccessModes[0] != actualPv.Spec.AccessModes[0]) || + expectedPv.Spec.CSI.Driver != actualPv.Spec.CSI.Driver { return false } - if pv1.Spec.CSI.NodeStageSecretRef != nil { - if pv2.Spec.CSI.NodeStageSecretRef == nil || pv1.Spec.CSI.NodeStageSecretRef.Name != pv2.Spec.CSI.NodeStageSecretRef.Name { + if expectedPv.Spec.CSI.NodeStageSecretRef != nil { + if actualPv.Spec.CSI.NodeStageSecretRef == nil || expectedPv.Spec.CSI.NodeStageSecretRef.Name != actualPv.Spec.CSI.NodeStageSecretRef.Name { return false } - } else if pv2.Spec.CSI.NodeStageSecretRef != nil { + } else if actualPv.Spec.CSI.NodeStageSecretRef != nil { return false } - if pv1.Spec.ClaimRef != nil { - if pv2.Spec.ClaimRef == nil { // ignore || pv1.Spec.ClaimRef.Name != pv2.Spec.ClaimRef.Name + if expectedPv.Spec.ClaimRef != nil { + if actualPv.Spec.ClaimRef == nil { // ignore || expectedPv.Spec.ClaimRef.Name != actualPv.Spec.ClaimRef.Name return false } - } else if pv2.Spec.ClaimRef != nil { + } else if actualPv.Spec.ClaimRef != nil { return false } return true diff --git a/pkg/apis/internal/persistentvolumeclaim.go b/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go similarity index 98% rename from pkg/apis/internal/persistentvolumeclaim.go rename to pkg/apis/internal/persistentvolume/persistentvolumeclaim.go index 1564a8087..be0e69d0c 100644 --- a/pkg/apis/internal/persistentvolumeclaim.go +++ b/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go @@ -1,4 +1,4 @@ -package internal +package persistentvolume import ( "github.com/equinor/radix-operator/pkg/apis/utils" From 51a9f068978eb551913a38155dbb682dc8412308 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 6 Dec 2024 16:28:36 +0100 Subject: [PATCH 15/68] Correcting unit-tests --- pkg/apis/deployment/volumemount.go | 23 ++- pkg/apis/deployment/volumemount_test.go | 241 ++++++++++++------------ 2 files changed, 137 insertions(+), 127 deletions(-) diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index db843e4c1..6a4289919 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -401,13 +401,13 @@ func (deploy *Deployment) garbageCollectVolumeMountsSecretsNoLongerInSpecForComp return deploy.GarbageCollectSecrets(ctx, secrets, excludeSecretNames) } -func getFunctionalCsiPersistentVolumesForNamespace(ctx context.Context, kubeClient kubernetes.Interface, namespace string) ([]corev1.PersistentVolume, error) { +func getCsiPersistentVolumesForNamespace(ctx context.Context, kubeClient kubernetes.Interface, namespace string, onlyFunctional bool) ([]corev1.PersistentVolume, error) { pvList, err := kubeClient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) if err != nil { return nil, err } return slice.FindAll(pvList.Items, func(pv corev1.PersistentVolume) bool { - return pvIsForCsiDriver(pv) && pvIsForNamespace(pv, namespace) && pvIsFunctional(pv) + return pvIsForCsiDriver(pv) && pvIsForNamespace(pv, namespace) && (!onlyFunctional || pvIsFunctional(pv)) }), nil } @@ -689,7 +689,7 @@ func (deploy *Deployment) deletePersistentVolumeClaim(ctx context.Context, names log.Ctx(ctx).Debug().Msgf("Skip deleting PVC - namespace %s or name %s is empty", namespace, pvcName) return nil } - if err := deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { + if err := deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil { // && !k8serrors.IsNotFound(err) { return err } return nil @@ -700,7 +700,7 @@ func (deploy *Deployment) deletePersistentVolume(ctx context.Context, pvName str log.Ctx(ctx).Debug().Msg("Skip deleting PersistentVolume - name is empty") return nil } - if err := deploy.kubeclient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { + if err := deploy.kubeclient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); err != nil { // && !k8serrors.IsNotFound(err) { return err } return nil @@ -764,7 +764,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, componentName string, identity *radixv1.Identity, desiredDeployment *appsv1.Deployment) error { namespace := deploy.radixDeployment.GetNamespace() - functionalPvList, err := getFunctionalCsiPersistentVolumesForNamespace(ctx, deploy.kubeclient, namespace) + functionalPvList, err := getCsiPersistentVolumesForNamespace(ctx, deploy.kubeclient, namespace, true) if err != nil { return err } @@ -861,6 +861,7 @@ func (deploy *Deployment) getCurrentlyUsedPersistentVolumeClaims(ctx context.Con for _, job := range jobsList.Items { pvcNames = appendUsedPersistenceVolumeClaimsFrom(pvcNames, job.Spec.Template.Spec.Volumes) } + // TODD add from RadixDeployments, connected to existing scheduled jobs and batches return appendUsedPersistenceVolumeClaimsFrom(pvcNames, desiredDeployment.Spec.Template.Spec.Volumes), nil } @@ -879,6 +880,8 @@ func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersist if err != nil { return err } + + !!!!! test failed - not deleted PVs var errs []error for _, pvc := range pvcList.Items { if _, ok := excludePvcNames[pvc.Name]; ok { @@ -898,6 +901,16 @@ func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersist if len(errs) > 0 { return errors.Join(errs...) } + listPvcs, err := deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + fmt.Printf("PersistentVolumes: %v\n", listPvcs) + listPvs, err := deploy.kubeclient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + fmt.Printf("PersistentVolumes: %v\n", listPvs) return nil } diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index df4f76f8b..5885f0e28 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -59,7 +59,7 @@ type volumeMountTestScenario struct { type deploymentVolumesTestScenario struct { name string - props expectedPvcScProperties + props expectedPvcPvProperties radixVolumeMounts []v1.RadixVolumeMount volumes []corev1.Volume existingPersistentVolumesBeforeTestRun []corev1.PersistentVolume @@ -412,7 +412,7 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { }) } -type expectedPvcScProperties struct { +type expectedPvcPvProperties struct { appName string environment string componentName string @@ -424,11 +424,10 @@ type expectedPvcScProperties struct { requestsVolumeMountSize string volumeAccessMode corev1.PersistentVolumeAccessMode volumeName string - scProvisioner string - scSecretName string - scTmpPath string - scGid string - scUid string + pvProvisioner string + pvSecretName string + pvGid string + pvUid string namespace string } @@ -508,7 +507,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { var scenarios []deploymentVolumesTestScenario // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ // name: "Create new volume", // props: props, @@ -532,108 +531,108 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // getScenario(getPropsCsiBlobVolume1Storage1(nil)), // } // }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - type scenarioProperties struct { - changedNewRadixVolumeName string - changedNewRadixVolumeStorageName string - expectedVolumeName string - expectedNewSecretName string - expectedNewPvcName string - expectedNewPvName string - } - getScenario := func(props expectedPvcScProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Update storage in existing volume name and storage", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.Name = scenarioProps.changedNewRadixVolumeName - vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) { - v.Name = scenarioProps.expectedVolumeName - }), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - }), - }, - existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), - }, - existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { - pv.ObjectMeta.Name = scenarioProps.expectedNewPvName - pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName - setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) - pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - changedNewRadixVolumeName: "volume101", - changedNewRadixVolumeStorageName: "storage101", - expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - expectedNewSecretName: "some-component-volume101-csiazurecreds", - expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - expectedNewPvName: "pv-radixvolumemount-some-uuid", - }), - } - }()...) // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { - // persistentVolumeForAnotherNamespace := createRandomPersistentVolume(props, utils.RandString(10), utils.RandString(10)) - // persistentVolumeForAnotherComponent := createRandomPersistentVolume(props, props.namespace, utils.RandString(10)) - // pvcForAnotherNamespace := createRandomPvc(props, utils.RandString(10), utils.RandString(10)) - // pvcForAnotherComponent := createRandomPvc(props, props.namespace, utils.RandString(10)) + // type scenarioProperties struct { + // changedNewRadixVolumeName string + // changedNewRadixVolumeStorageName string + // expectedVolumeName string + // expectedNewSecretName string + // expectedNewPvcName string + // expectedNewPvName string + // } + // getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ - // name: "Garbage collect orphaned PVCs and PersistentVolume", + // name: "Update storage in existing volume name and storage", // props: props, // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.Name = scenarioProps.changedNewRadixVolumeName + // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + // }), // }, // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), + // createTestVolume(props, func(v *corev1.Volume) { + // v.Name = scenarioProps.expectedVolumeName + // }), // }, // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), // }, // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + // }), // }, // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPersistentVolume(props, props.namespace, props.componentName), - // persistentVolumeForAnotherNamespace, - // persistentVolumeForAnotherComponent, + // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), // }, // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), - // persistentVolumeForAnotherNamespace, - // persistentVolumeForAnotherComponent, + // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName + // setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + // pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + // }), // }, // } // } // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + // changedNewRadixVolumeName: "volume101", + // changedNewRadixVolumeStorageName: "storage101", + // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + // expectedNewSecretName: "some-component-volume101-csiazurecreds", + // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + // expectedNewPvName: "pv-radixvolumemount-some-uuid", + // }), // } // }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + persistentVolumeForAnotherNamespace := createRandomPersistentVolume(props, utils.RandString(10), utils.RandString(10)) + persistentVolumeForAnotherComponent := createRandomPersistentVolume(props, props.namespace, utils.RandString(10)) + pvcForAnotherNamespace := createRandomPvc(props, utils.RandString(10), utils.RandString(10)) + pvcForAnotherComponent := createRandomPvc(props, props.namespace, utils.RandString(10)) + return deploymentVolumesTestScenario{ + name: "Garbage collect orphaned PVCs and PersistentVolume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createRandomPvc(props, props.namespace, props.componentName), + pvcForAnotherNamespace, + pvcForAnotherComponent, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherNamespace, + pvcForAnotherComponent, + }, + existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + createRandomPersistentVolume(props, props.namespace, props.componentName), + persistentVolumeForAnotherNamespace, + persistentVolumeForAnotherComponent, + }, + existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), + persistentVolumeForAnotherNamespace, + persistentVolumeForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ // name: "Set readonly volume", // props: props, @@ -666,7 +665,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // } // }()...) // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ // name: "Set ReadWriteOnce volume", // props: props, @@ -697,7 +696,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // } // }()...) // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ // name: "Set ReadWriteMany volume", // props: props, @@ -728,7 +727,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // } // }()...) // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", // props: props, @@ -767,7 +766,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }()...) // // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", // props: props, @@ -819,7 +818,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }()...) // // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcScProperties) deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ // name: "Create new BlobFuse2 volume has disabled streaming", // props: props, @@ -1103,7 +1102,7 @@ func Test_EmptyDir(t *testing.T) { assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2) } -func createRandomPersistentVolume(props expectedPvcScProperties, namespace, componentName string) corev1.PersistentVolume { +func createRandomPersistentVolume(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolume { return createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { pv.ObjectMeta.Name = utils.RandString(10) pv.ObjectMeta.Labels[kube.RadixNamespace] = namespace @@ -1111,7 +1110,7 @@ func createRandomPersistentVolume(props expectedPvcScProperties, namespace, comp }) } -func createRandomPvc(props expectedPvcScProperties, namespace, componentName string) corev1.PersistentVolumeClaim { +func createRandomPvc(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolumeClaim { return createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { pvc.ObjectMeta.Name = utils.RandString(10) pvc.ObjectMeta.Namespace = namespace @@ -1132,11 +1131,11 @@ func createRandomPvc(props expectedPvcScProperties, namespace, componentName str // fmt.Printf("MountOption %s not found for the storage class", key) // } -func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcScProperties)) expectedPvcScProperties { +func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcPvProperties)) expectedPvcPvProperties { appName := "any-app" environment := "some-env" componentName := "some-component" - props := expectedPvcScProperties{ + props := expectedPvcPvProperties{ appName: appName, environment: environment, namespace: fmt.Sprintf("%s-%s", appName, environment), @@ -1149,11 +1148,10 @@ func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcScProperties)) expec requestsVolumeMountSize: "1Mi", volumeAccessMode: corev1.ReadOnlyMany, // default access mode volumeName: "csi-az-blob-some-component-volume1-storage1", - scProvisioner: provisionerBlobCsiAzure, - scSecretName: "some-component-volume1-csiazurecreds", - scTmpPath: "/tmp/any-app-some-env/csi-az-blob/some-component/volume1/storage1", - scGid: "1000", - scUid: "", + pvProvisioner: provisionerBlobCsiAzure, + pvSecretName: "some-component-volume1-csiazurecreds", + pvGid: "1000", + pvUid: "", } if modify != nil { modify(&props) @@ -1161,11 +1159,11 @@ func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcScProperties)) expec return props } -func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcScProperties)) expectedPvcScProperties { +func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcPvProperties)) expectedPvcPvProperties { appName := "any-app" environment := "some-env" componentName := "some-component" - props := expectedPvcScProperties{ + props := expectedPvcPvProperties{ appName: appName, environment: environment, namespace: fmt.Sprintf("%s-%s", appName, environment), @@ -1178,11 +1176,10 @@ func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcScProperties)) requestsVolumeMountSize: "1Mi", volumeAccessMode: corev1.ReadOnlyMany, // default access mode volumeName: "csi-blobfuse2-fuse2-some-component-volume1-storage1", - scProvisioner: provisionerBlobCsiAzure, - scSecretName: "some-component-volume1-csiazurecreds", - scTmpPath: "/tmp/any-app-some-env/csi-blobfuse2-fuse2/some-component/volume1/storage1", - scGid: "1000", - scUid: "", + pvProvisioner: provisionerBlobCsiAzure, + pvSecretName: "some-component-volume1-csiazurecreds", + pvGid: "1000", + pvUid: "", } if modify != nil { modify(&props) @@ -1280,7 +1277,7 @@ func buildRdWithComponentBuilders(appName string, environment string, componentB BuildRD() } -func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *corev1.PersistentVolume)) corev1.PersistentVolume { +func createExpectedVolumeMount(props expectedPvcPvProperties, modify func(pv *corev1.PersistentVolume)) corev1.PersistentVolume { mountOptions := []string{ "--file-cache-timeout-in-seconds=120", "--use-attr-cache=true", @@ -1306,7 +1303,7 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co }, Annotations: map[string]string{ persistentvolume.CsiAnnotationProvisionedBy: provisionerBlobCsiAzure, - persistentvolume.CsiAnnotationProvisionerDeletionSecretName: props.scSecretName, + persistentvolume.CsiAnnotationProvisionerDeletionSecretName: props.pvSecretName, persistentvolume.CsiAnnotationProvisionerDeletionSecretNamespace: props.namespace, }, }, @@ -1326,7 +1323,7 @@ func createExpectedVolumeMount(props expectedPvcScProperties, modify func(pv *co // skip auto-created by the provisioner "storage.kubernetes.io/csiProvisionerIdentity": "1732528668611-2190-blob.csi.azure.com" }, NodeStageSecretRef: &corev1.SecretReference{ - Name: props.scSecretName, + Name: props.pvSecretName, Namespace: props.namespace, }, }, @@ -1361,17 +1358,17 @@ func setVolumeMountAttribute(pv *corev1.PersistentVolume, radixVolumeMountType v } } -func getPersistentVolumeIdMountOption(props expectedPvcScProperties) string { - if len(props.scGid) > 0 { - return fmt.Sprintf("-o gid=%s", props.scGid) +func getPersistentVolumeIdMountOption(props expectedPvcPvProperties) string { + if len(props.pvGid) > 0 { + return fmt.Sprintf("-o gid=%s", props.pvGid) } - if len(props.scUid) > 0 { - return fmt.Sprintf("-o uid=%s", props.scGid) + if len(props.pvUid) > 0 { + return fmt.Sprintf("-o uid=%s", props.pvGid) } return "" } -func createExpectedPvc(props expectedPvcScProperties, modify func(*corev1.PersistentVolumeClaim)) corev1.PersistentVolumeClaim { +func createExpectedPvc(props expectedPvcPvProperties, modify func(*corev1.PersistentVolumeClaim)) corev1.PersistentVolumeClaim { labels := map[string]string{ kube.RadixAppLabel: props.appName, kube.RadixComponentLabel: props.componentName, @@ -1398,7 +1395,7 @@ func createExpectedPvc(props expectedPvcScProperties, modify func(*corev1.Persis return pvc } -func createTestVolume(pvcProps expectedPvcScProperties, modify func(*corev1.Volume)) corev1.Volume { +func createTestVolume(pvcProps expectedPvcPvProperties, modify func(*corev1.Volume)) corev1.Volume { volume := corev1.Volume{ Name: pvcProps.volumeName, VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ @@ -1411,7 +1408,7 @@ func createTestVolume(pvcProps expectedPvcScProperties, modify func(*corev1.Volu return volume } -func createRadixVolumeMount(props expectedPvcScProperties, modify func(mount *v1.RadixVolumeMount)) v1.RadixVolumeMount { +func createRadixVolumeMount(props expectedPvcPvProperties, modify func(mount *v1.RadixVolumeMount)) v1.RadixVolumeMount { volumeMount := v1.RadixVolumeMount{ Type: props.radixVolumeMountType, Name: props.radixVolumeMountName, @@ -1424,7 +1421,7 @@ func createRadixVolumeMount(props expectedPvcScProperties, modify func(mount *v1 } return volumeMount } -func createBlobFuse2RadixVolumeMount(props expectedPvcScProperties, modify func(mount *v1.RadixVolumeMount)) v1.RadixVolumeMount { +func createBlobFuse2RadixVolumeMount(props expectedPvcPvProperties, modify func(mount *v1.RadixVolumeMount)) v1.RadixVolumeMount { volumeMount := v1.RadixVolumeMount{ Name: props.radixVolumeMountName, Path: "path1", From 38ee3f4b2e47cd73206c5dcef7e27f7bc688ae74 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 9 Dec 2024 11:28:51 +0100 Subject: [PATCH 16/68] Correcting unit-tests --- pkg/apis/deployment/volumemount.go | 19 +-- pkg/apis/deployment/volumemount_test.go | 121 ++++++++++-------- .../persistentvolume/persistentvolume.go | 6 +- 3 files changed, 75 insertions(+), 71 deletions(-) diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index 6a4289919..e4cef3d57 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -700,7 +700,7 @@ func (deploy *Deployment) deletePersistentVolume(ctx context.Context, pvName str log.Ctx(ctx).Debug().Msg("Skip deleting PersistentVolume - name is empty") return nil } - if err := deploy.kubeclient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); err != nil { // && !k8serrors.IsNotFound(err) { + if err := deploy.kubeclient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { return err } return nil @@ -748,7 +748,7 @@ func knownCSIDriver(driver string) bool { // createOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims, PersistentVolume func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Context, deployComponent radixv1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) error { - componentName := desiredDeployment.ObjectMeta.Name + componentName := deployComponent.GetName() if err := deploy.createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx, componentName, deployComponent.GetIdentity(), desiredDeployment); err != nil { return err } @@ -759,7 +759,8 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Cont if err = deploy.garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx, componentName, currentlyUsedPvcNames); err != nil { return err } - return deploy.garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, currentlyUsedPvcNames) + err = deploy.garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, currentlyUsedPvcNames) + return err } func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, componentName string, identity *radixv1.Identity, desiredDeployment *appsv1.Deployment) error { @@ -880,8 +881,6 @@ func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersist if err != nil { return err } - - !!!!! test failed - not deleted PVs var errs []error for _, pvc := range pvcList.Items { if _, ok := excludePvcNames[pvc.Name]; ok { @@ -901,16 +900,6 @@ func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersist if len(errs) > 0 { return errors.Join(errs...) } - listPvcs, err := deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - return err - } - fmt.Printf("PersistentVolumes: %v\n", listPvcs) - listPvs, err := deploy.kubeclient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) - if err != nil { - return err - } - fmt.Printf("PersistentVolumes: %v\n", listPvs) return nil } diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 5885f0e28..cd5db86d2 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -58,14 +58,14 @@ type volumeMountTestScenario struct { } type deploymentVolumesTestScenario struct { - name string - props expectedPvcPvProperties - radixVolumeMounts []v1.RadixVolumeMount - volumes []corev1.Volume - existingPersistentVolumesBeforeTestRun []corev1.PersistentVolume - existingPvcsBeforeTestRun []corev1.PersistentVolumeClaim - existingPersistentVolumeAfterTestRun []corev1.PersistentVolume - existingPvcsAfterTestRun []corev1.PersistentVolumeClaim + name string + props expectedPvcPvProperties + radixVolumeMounts []v1.RadixVolumeMount + volumes []corev1.Volume + existingPVsBeforeTestRun []corev1.PersistentVolume + existingPvcsBeforeTestRun []corev1.PersistentVolumeClaim + existingPVsAfterTestRun []corev1.PersistentVolume + existingPvcsAfterTestRun []corev1.PersistentVolumeClaim } type pvcTestScenario struct { @@ -501,9 +501,13 @@ func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { } func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { - appName := "any-app" - environment := "some-env" - componentName := "some-component" + const ( + appName = "any-app" + environment = "some-env" + componentName = "some-component" + ) + anotherNamespace := commonUtils.RandString(10) + anotherComponentName := commonUtils.RandString(10) var scenarios []deploymentVolumesTestScenario // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { @@ -521,8 +525,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), // }, - // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, - // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), // }, // } @@ -565,10 +569,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName // }), // }, - // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), // }, - // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // existingPVsAfterTestRun: []corev1.PersistentVolume{ // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName @@ -592,10 +596,14 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - persistentVolumeForAnotherNamespace := createRandomPersistentVolume(props, utils.RandString(10), utils.RandString(10)) - persistentVolumeForAnotherComponent := createRandomPersistentVolume(props, props.namespace, utils.RandString(10)) - pvcForAnotherNamespace := createRandomPvc(props, utils.RandString(10), utils.RandString(10)) - pvcForAnotherComponent := createRandomPvc(props, props.namespace, utils.RandString(10)) + pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) + pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) + pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + pvForAnotherNamespace.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvcForAnotherNamespace.Name + pvForAnotherComponent.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvcForAnotherComponent.Name + pvcForAnotherNamespace.Spec.VolumeName = pvForAnotherNamespace.Name + pvcForAnotherComponent.Spec.VolumeName = pvForAnotherComponent.Name return deploymentVolumesTestScenario{ name: "Garbage collect orphaned PVCs and PersistentVolume", props: props, @@ -615,15 +623,15 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { pvcForAnotherNamespace, pvcForAnotherComponent, }, - existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - createRandomPersistentVolume(props, props.namespace, props.componentName), - persistentVolumeForAnotherNamespace, - persistentVolumeForAnotherComponent, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createRandomPv(props, props.namespace, props.componentName), + pvForAnotherNamespace, + pvForAnotherComponent, }, - existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + existingPVsAfterTestRun: []corev1.PersistentVolume{ createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), - persistentVolumeForAnotherNamespace, - persistentVolumeForAnotherComponent, + pvForAnotherNamespace, + pvForAnotherComponent, }, } } @@ -650,10 +658,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} // }), // }, - // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPersistentVolume(props, props.namespace, props.componentName), + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPv(props, props.namespace, props.componentName), // }, - // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // existingPVsAfterTestRun: []corev1.PersistentVolume{ // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") // }), @@ -683,10 +691,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} // }), // }, - // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPersistentVolume(props, props.namespace, props.componentName), + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPv(props, props.namespace, props.componentName), // }, - // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // existingPVsAfterTestRun: []corev1.PersistentVolume{ // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), // }, // } @@ -714,10 +722,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} // }), // }, - // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPersistentVolume(props, props.namespace, props.componentName), + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPv(props, props.namespace, props.componentName), // }, - // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // existingPVsAfterTestRun: []corev1.PersistentVolume{ // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), // }, // } @@ -741,8 +749,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), // }, - // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, - // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { // pv.Spec.MountOptions = []string{ // "--file-cache-timeout-in-seconds=120", @@ -788,8 +796,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), // }, - // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, - // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { // pv.Spec.MountOptions = []string{ // "--file-cache-timeout-in-seconds=120", @@ -841,8 +849,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), // }, - // existingPersistentVolumesBeforeTestRun: []corev1.PersistentVolume{}, - // existingPersistentVolumeAfterTestRun: []corev1.PersistentVolume{ + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { // pv.Spec.MountOptions = []string{ // "--file-cache-timeout-in-seconds=120", @@ -880,10 +888,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), &deployComponent, desiredDeployment) assert.Nil(t, err) - existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment, utils.GetEnvironmentNamespace(appName, environment)) + existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment) assert.Nil(t, err) assert.True(t, equalPersistentVolumeClaims(&scenario.existingPvcsAfterTestRun, &existingPvcs), "PVC-s are not equal") - assert.True(t, equalPersistentVolumes(&scenario.existingPersistentVolumeAfterTestRun, &existingPvs), "PV-s are not equal") + assert.True(t, equalPersistentVolumes(&scenario.existingPVsAfterTestRun, &existingPvs), "PV-s are not equal") } } }) @@ -1102,20 +1110,27 @@ func Test_EmptyDir(t *testing.T) { assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2) } -func createRandomPersistentVolume(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolume { +func createRandomPv(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolume { return createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { - pv.ObjectMeta.Name = utils.RandString(10) + pvName := getCsiAzurePersistentVolumeName() + pv.ObjectMeta.Name = pvName pv.ObjectMeta.Labels[kube.RadixNamespace] = namespace pv.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName + pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvName] = pvName }) } func createRandomPvc(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolumeClaim { return createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.ObjectMeta.Name = utils.RandString(10) + pvcName, err := getCsiAzurePersistentVolumeClaimName(componentName, &v1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: v1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.radixStorageName, Path: "/tmp"}) + if err != nil { + panic(err) + } + pvName := getCsiAzurePersistentVolumeName() + pvc.ObjectMeta.Name = pvcName pvc.ObjectMeta.Namespace = namespace pvc.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName - pvc.Spec.VolumeName = utils.RandString(10) + pvc.Spec.VolumeName = pvName }) } @@ -1191,15 +1206,15 @@ func putExistingDeploymentVolumesScenarioDataToFakeCluster(scenario *deploymentV for _, pvc := range scenario.existingPvcsBeforeTestRun { _, _ = deployment.kubeclient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.Background(), &pvc, metav1.CreateOptions{}) } - for _, pv := range scenario.existingPersistentVolumesBeforeTestRun { + for _, pv := range scenario.existingPVsBeforeTestRun { _, _ = deployment.kubeclient.CoreV1().PersistentVolumes().Create(context.Background(), &pv, metav1.CreateOptions{}) } } -func getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment *Deployment, namespace string) ([]corev1.PersistentVolumeClaim, []corev1.PersistentVolume, error) { +func getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment *Deployment) ([]corev1.PersistentVolumeClaim, []corev1.PersistentVolume, error) { var pvcItems []corev1.PersistentVolumeClaim var pvItems []corev1.PersistentVolume - pvcList, err := deployment.kubeclient.CoreV1().PersistentVolumeClaims(namespace).List(context.Background(), metav1.ListOptions{}) + pvcList, err := deployment.kubeclient.CoreV1().PersistentVolumeClaims("").List(context.Background(), metav1.ListOptions{}) if err != nil { return nil, nil, err } @@ -1253,13 +1268,13 @@ func createPvc(namespace, componentName string, mountType v1.MountType, modify f appName := "app" pvc := corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: utils.RandString(10), // Set in test scenario + Name: commonUtils.RandString(10), // Set in test scenario Namespace: namespace, Labels: map[string]string{ kube.RadixAppLabel: appName, kube.RadixComponentLabel: componentName, kube.RadixMountTypeLabel: string(mountType), - kube.RadixVolumeMountNameLabel: utils.RandString(10), // Set in test scenario + kube.RadixVolumeMountNameLabel: commonUtils.RandString(10), // Set in test scenario }, }, } @@ -1339,6 +1354,7 @@ func createExpectedVolumeMount(props expectedPvcPvProperties, modify func(pv *co MountOptions: mountOptions, PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, }, + Status: corev1.PersistentVolumeStatus{Phase: corev1.VolumeBound}, } setVolumeMountAttribute(&pv, props.radixVolumeMountType, props.radixStorageName, props.pvcName) if modify != nil { @@ -1388,6 +1404,7 @@ func createExpectedPvc(props expectedPvcPvProperties, modify func(*corev1.Persis }, VolumeName: props.persistentVolumeName, }, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } if modify != nil { modify(&pvc) diff --git a/pkg/apis/internal/persistentvolume/persistentvolume.go b/pkg/apis/internal/persistentvolume/persistentvolume.go index eff16a17a..f14e75689 100644 --- a/pkg/apis/internal/persistentvolume/persistentvolume.go +++ b/pkg/apis/internal/persistentvolume/persistentvolume.go @@ -72,14 +72,12 @@ func EqualPersistentVolumesForTest(expectedPv, actualPv *corev1.PersistentVolume if len(expectedNameAttr) == 0 || len(actualNameAttr) == 0 { return false } - s := expectedNameAttr[:20] - if s != actualNameAttr[:20] { + if expectedNameAttr[:20] != actualNameAttr[:20] { return false } expectedPvcNameAttr := expectedPv.Spec.CSI.VolumeAttributes[CsiVolumeMountAttributePvcName] actualPvcNameAttr := actualPv.Spec.CSI.VolumeAttributes[CsiVolumeMountAttributePvcName] - s2 := expectedPvcNameAttr[:len(expectedPvcNameAttr)-5] - if s2 != actualPvcNameAttr[:len(actualPvcNameAttr)-5] { + if expectedPvcNameAttr[:len(expectedPvcNameAttr)-5] != actualPvcNameAttr[:len(actualPvcNameAttr)-5] { return false } From cb6f145e9d2444f844f8e784f68c704bd38342d7 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 9 Dec 2024 13:55:05 +0100 Subject: [PATCH 17/68] Correcting unit-tests --- pkg/apis/deployment/volumemount_test.go | 201 ++++++++++-------- .../persistentvolume/persistentvolumeclaim.go | 2 +- 2 files changed, 108 insertions(+), 95 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index cd5db86d2..741afadab 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -500,14 +500,23 @@ func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { }) } +func modifyPvc(pvc corev1.PersistentVolumeClaim, modify func(pvc *corev1.PersistentVolumeClaim)) corev1.PersistentVolumeClaim { + modify(&pvc) + return pvc +} +func modifyPv(pv corev1.PersistentVolume, modify func(pv *corev1.PersistentVolume)) corev1.PersistentVolume { + modify(&pv) + return pv +} + func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { const ( appName = "any-app" environment = "some-env" componentName = "some-component" ) - anotherNamespace := commonUtils.RandString(10) - anotherComponentName := commonUtils.RandString(10) + // anotherNamespace := commonUtils.RandString(10) + // anotherComponentName := commonUtils.RandString(10) var scenarios []deploymentVolumesTestScenario // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { @@ -527,7 +536,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }, // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), // }, // } // } @@ -570,10 +579,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }), // }, // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), // }, // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName // pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName @@ -594,77 +603,42 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }), // } // }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) - pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) - pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - pvForAnotherNamespace.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvcForAnotherNamespace.Name - pvForAnotherComponent.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvcForAnotherComponent.Name - pvcForAnotherNamespace.Spec.VolumeName = pvForAnotherNamespace.Name - pvcForAnotherComponent.Spec.VolumeName = pvForAnotherComponent.Name - return deploymentVolumesTestScenario{ - name: "Garbage collect orphaned PVCs and PersistentVolume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - createRandomPv(props, props.namespace, props.componentName), - pvForAnotherNamespace, - pvForAnotherComponent, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), - pvForAnotherNamespace, - pvForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) + // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + // pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) + // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + // matchPvAndPvc(pvForAnotherNamespace, pvcForAnotherNamespace) + // matchPvAndPvc(pvForAnotherComponent, pvcForAnotherNamespace) // return deploymentVolumesTestScenario{ - // name: "Set readonly volume", + // name: "Garbage collect orphaned PVCs and PersistentVolume", // props: props, // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), // }, // volumes: []corev1.Volume{ // createTestVolume(props, func(v *corev1.Volume) {}), // }, // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ // createRandomPvc(props, props.namespace, props.componentName), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, // }, // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // }), + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, // }, // existingPVsBeforeTestRun: []corev1.PersistentVolume{ // createRandomPv(props, props.namespace, props.componentName), + // pvForAnotherNamespace, + // pvForAnotherComponent, // }, // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - // }), + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // pvForAnotherNamespace, + // pvForAnotherComponent, // }, // } // } @@ -675,10 +649,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteOnce volume", + // name: "Set readonly volume", // props: props, // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), // }, // volumes: []corev1.Volume{ // createTestVolume(props, func(v *corev1.Volume) {}), @@ -688,14 +662,16 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }, // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} // }), // }, // existingPVsBeforeTestRun: []corev1.PersistentVolume{ // createRandomPv(props, props.namespace, props.componentName), // }, // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") + // }), // }, // } // } @@ -703,6 +679,43 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // getScenario(getPropsCsiBlobVolume1Storage1(nil)), // } // }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteOnce volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ @@ -726,7 +739,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // createRandomPv(props, props.namespace, props.componentName), // }, // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) {}), + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), // }, // } // } @@ -751,7 +764,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }, // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { // pv.Spec.MountOptions = []string{ // "--file-cache-timeout-in-seconds=120", // "--use-attr-cache=true", @@ -798,7 +811,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }, // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { // pv.Spec.MountOptions = []string{ // "--file-cache-timeout-in-seconds=120", // "--use-attr-cache=true", @@ -851,7 +864,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }, // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { // pv.Spec.MountOptions = []string{ // "--file-cache-timeout-in-seconds=120", // "--use-attr-cache=true", @@ -897,6 +910,11 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }) } +func matchPvAndPvc(pv *corev1.PersistentVolume, pvc *corev1.PersistentVolumeClaim) { + pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvc.Name + pvc.Spec.VolumeName = pv.Name +} + func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureKeyVaultResources() { appName := "app" namespace := "some-namespace" @@ -1111,7 +1129,7 @@ func Test_EmptyDir(t *testing.T) { } func createRandomPv(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolume { - return createExpectedVolumeMount(props, func(pv *corev1.PersistentVolume) { + return createExpectedPv(props, func(pv *corev1.PersistentVolume) { pvName := getCsiAzurePersistentVolumeName() pv.ObjectMeta.Name = pvName pv.ObjectMeta.Labels[kube.RadixNamespace] = namespace @@ -1134,18 +1152,6 @@ func createRandomPvc(props expectedPvcPvProperties, namespace, componentName str }) } -// TODO: this option does not work with blobfuse2 in some reason - investigate to make use separate disk volume for csi volumes -// func setPersistentVolumeMountOption(pv *corev1.PersistentVolume, key, value string) { -// mountOptions := pv.MountOptions -// for i, option := range mountOptions { -// if strings.Contains(option, key) { -// mountOptions[i] = fmt.Sprintf("%s=%s", key, value) -// return -// } -// } -// fmt.Printf("MountOption %s not found for the storage class", key) -// } - func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcPvProperties)) expectedPvcPvProperties { appName := "any-app" environment := "some-env" @@ -1292,21 +1298,8 @@ func buildRdWithComponentBuilders(appName string, environment string, componentB BuildRD() } -func createExpectedVolumeMount(props expectedPvcPvProperties, modify func(pv *corev1.PersistentVolume)) corev1.PersistentVolume { - mountOptions := []string{ - "--file-cache-timeout-in-seconds=120", - "--use-attr-cache=true", - "--cancel-list-on-mount-seconds=0", - "-o allow_other", - "-o attr_timeout=120", - "-o entry_timeout=120", - "-o negative_timeout=120", - "-o ro", - } - idOption := getPersistentVolumeIdMountOption(props) - if len(idOption) > 0 { - mountOptions = append(mountOptions, idOption) - } +func createExpectedPv(props expectedPvcPvProperties, modify func(pv *corev1.PersistentVolume)) corev1.PersistentVolume { + mountOptions := getMountOptions(props, true) pv := corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: props.persistentVolumeName, @@ -1363,6 +1356,26 @@ func createExpectedVolumeMount(props expectedPvcPvProperties, modify func(pv *co return pv } +func getMountOptions(props expectedPvcPvProperties, readOnly bool) []string { + options := []string{ + "--file-cache-timeout-in-seconds=120", + "--use-attr-cache=true", + "--cancel-list-on-mount-seconds=0", + "-o allow_other", + "-o attr_timeout=120", + "-o entry_timeout=120", + "-o negative_timeout=120", + } + if readOnly { + options = append(options, "-o ro") + } + idOption := getPersistentVolumeIdMountOption(props) + if len(idOption) > 0 { + options = append(options, idOption) + } + return options +} + func setVolumeMountAttribute(pv *corev1.PersistentVolume, radixVolumeMountType v1.MountType, containerName, pvcName string) { pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributeContainerName] = containerName pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvcName diff --git a/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go b/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go index be0e69d0c..724643b8e 100644 --- a/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go +++ b/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go @@ -33,7 +33,7 @@ func EqualPersistentVolumeClaims(pvc1, pvc2 *corev1.PersistentVolumeClaim) bool // ignore pvc1.Spec.StorageClassName != pvc2.Spec.StorageClassName for transition period if pvc1.Spec.Resources.Requests[corev1.ResourceStorage] != pvc2.Spec.Resources.Requests[corev1.ResourceStorage] || len(pvc1.Spec.AccessModes) != len(pvc2.Spec.AccessModes) || - (len(pvc1.Spec.AccessModes) != 1 && pvc1.Spec.AccessModes[0] != pvc2.Spec.AccessModes[0]) || + (len(pvc1.Spec.AccessModes) == 1 && pvc1.Spec.AccessModes[0] != pvc2.Spec.AccessModes[0]) || pvc1.Spec.VolumeMode != pvc2.Spec.VolumeMode { return false } From 119618e27116ddce5f025ab9730f8ba06c298a06 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 9 Dec 2024 13:57:49 +0100 Subject: [PATCH 18/68] Correcting unit-tests --- pkg/apis/deployment/volumemount_test.go | 76 +++++++++++++------------ 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 741afadab..9b6aa82c3 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -679,16 +679,53 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // getScenario(getPropsCsiBlobVolume1Storage1(nil)), // } // }()...) + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteOnce volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // existingPvc, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + // }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { existingPvc := createExpectedPvc(props, nil) existingPv := createExpectedPv(props, nil) matchPvAndPvc(&existingPv, &existingPvc) return deploymentVolumesTestScenario{ - name: "Set ReadWriteOnce volume", + name: "Set ReadWriteMany volume", props: props, radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), }, volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), @@ -698,7 +735,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }, existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} }), }, existingPVsBeforeTestRun: []corev1.PersistentVolume{ @@ -706,7 +743,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }, existingPVsAfterTestRun: []corev1.PersistentVolume{ modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} pv.Spec.MountOptions = getMountOptions(props, false) }), }, @@ -719,37 +756,6 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteMany volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPv(props, props.namespace, props.componentName), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - // }()...) - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", // props: props, // radixVolumeMounts: []v1.RadixVolumeMount{ From 6bc9ef70a710b3ffe6eb62c1247355a5319a4dce Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 9 Dec 2024 13:59:40 +0100 Subject: [PATCH 19/68] Correcting unit-tests --- pkg/apis/deployment/volumemount_test.go | 56 +++++++++++++++++++++---- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 9b6aa82c3..5a33e6180 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -716,37 +716,79 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // getScenario(getPropsCsiBlobVolume1Storage1(nil)), // } // }()...) + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteMany volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // existingPvc, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + // }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { existingPvc := createExpectedPvc(props, nil) existingPv := createExpectedPv(props, nil) matchPvAndPvc(&existingPv, &existingPvc) return deploymentVolumesTestScenario{ - name: "Set ReadWriteMany volume", + name: "Set ReadOnlyMany volume", props: props, radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), }, volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - existingPvc, + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), }, existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} }), }, existingPVsBeforeTestRun: []corev1.PersistentVolume{ - existingPv, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ modifyPv(existingPv, func(pv *corev1.PersistentVolume) { pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} pv.Spec.MountOptions = getMountOptions(props, false) }), }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + pv.Spec.MountOptions = getMountOptions(props, true) + }), + }, } } return []deploymentVolumesTestScenario{ From cbff1800060165f9fb92c665e07c32e5129022ed Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 9 Dec 2024 14:28:51 +0100 Subject: [PATCH 20/68] Correcting unit-tests --- pkg/apis/deployment/volumemount_test.go | 117 +++++++++++------------- 1 file changed, 53 insertions(+), 64 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 5a33e6180..db0b649a0 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -753,87 +753,76 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // getScenario(getPropsCsiBlobVolume1Storage1(nil)), // } // }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadOnlyMany volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - }), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - pv.Spec.MountOptions = getMountOptions(props, true) - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + // name: "Set ReadOnlyMany volume", // props: props, // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), // }, // volumes: []corev1.Volume{ // createTestVolume(props, func(v *corev1.Volume) {}), // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // }), + // }, // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = []string{ - // "--file-cache-timeout-in-seconds=120", - // "--use-attr-cache=true", - // "--cancel-list-on-mount-seconds=0", - // "-o allow_other", - // "-o attr_timeout=120", - // "-o entry_timeout=120", - // "-o negative_timeout=120", - // "-o gid=1000", - // "--streaming=true", - // "--use-adls=false", - // } + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // pv.Spec.MountOptions = getMountOptions(props, true) // }), // }, // } // } // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), // } // }()...) - // + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, false, "--streaming=true", "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ @@ -1404,7 +1393,7 @@ func createExpectedPv(props expectedPvcPvProperties, modify func(pv *corev1.Pers return pv } -func getMountOptions(props expectedPvcPvProperties, readOnly bool) []string { +func getMountOptions(props expectedPvcPvProperties, readOnly bool, extraOptions ...string) []string { options := []string{ "--file-cache-timeout-in-seconds=120", "--use-attr-cache=true", @@ -1421,7 +1410,7 @@ func getMountOptions(props expectedPvcPvProperties, readOnly bool) []string { if len(idOption) > 0 { options = append(options, idOption) } - return options + return append(options, extraOptions...) } func setVolumeMountAttribute(pv *corev1.PersistentVolume, radixVolumeMountType v1.MountType, containerName, pvcName string) { From 996237090f83a2672dae17bcea1b30571faffd9c Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 9 Dec 2024 15:26:54 +0100 Subject: [PATCH 21/68] Correcting unit-tests --- pkg/apis/deployment/volumemount_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index db0b649a0..a3f23dd13 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -813,7 +813,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPVsBeforeTestRun: []corev1.PersistentVolume{}, existingPVsAfterTestRun: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, false, "--streaming=true", "--use-adls=false") + pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") }), }, } From 0e892d43909fbc1bdb6460d4e62d1c56d0c9f23d Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 9 Dec 2024 15:29:34 +0100 Subject: [PATCH 22/68] Correcting unit-tests --- pkg/apis/deployment/volumemount_test.go | 100 +++++++++++------------- 1 file changed, 45 insertions(+), 55 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index a3f23dd13..bff99d370 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -795,49 +795,13 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // getScenario(getPropsCsiBlobVolume1Storage1(nil)), // } // }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", // props: props, // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - // StreamCache: pointers.Ptr(uint64(101)), - // BlockSize: pointers.Ptr(uint64(102)), - // BufferSize: pointers.Ptr(uint64(103)), - // MaxBuffers: pointers.Ptr(uint64(104)), - // MaxBlocksPerFile: pointers.Ptr(uint64(105)), - // } - // }), + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), // }, // volumes: []corev1.Volume{ // createTestVolume(props, func(v *corev1.Volume) {}), @@ -849,23 +813,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, // existingPVsAfterTestRun: []corev1.PersistentVolume{ // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = []string{ - // "--file-cache-timeout-in-seconds=120", - // "--use-attr-cache=true", - // "--cancel-list-on-mount-seconds=0", - // "-o allow_other", - // "-o attr_timeout=120", - // "-o entry_timeout=120", - // "-o negative_timeout=120", - // "-o gid=1000", - // "--streaming=true", - // "--stream-cache-mb=101", - // "--block-size-mb=102", - // "--buffer-size-mb=103", - // "--max-buffers=104", - // "--max-blocks-per-file=105", - // "--use-adls=false", - // } + // pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") // }), // }, // } @@ -874,6 +822,48 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), // } // }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, + "--streaming=true", + "--stream-cache-mb=101", + "--block-size-mb=102", + "--buffer-size-mb=103", + "--max-buffers=104", + "--max-blocks-per-file=105", + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) // // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { From bf3d0a9d0823fdc35db9863e7f0e10736321edce Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 9 Dec 2024 15:31:56 +0100 Subject: [PATCH 23/68] Correcting unit-tests --- pkg/apis/deployment/volumemount_test.go | 99 +++++++++++-------------- 1 file changed, 45 insertions(+), 54 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index bff99d370..d2904ec17 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -822,14 +822,58 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), // } // }()...) + // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + // StreamCache: pointers.Ptr(uint64(101)), + // BlockSize: pointers.Ptr(uint64(102)), + // BufferSize: pointers.Ptr(uint64(103)), + // MaxBuffers: pointers.Ptr(uint64(104)), + // MaxBlocksPerFile: pointers.Ptr(uint64(105)), + // } + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, + // "--streaming=true", + // "--stream-cache-mb=101", + // "--block-size-mb=102", + // "--buffer-size-mb=103", + // "--max-buffers=104", + // "--max-blocks-per-file=105", + // "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + // }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + name: "Create new BlobFuse2 volume has disabled streaming", props: props, radixVolumeMounts: []v1.RadixVolumeMount{ createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + Enabled: pointers.Ptr(false), StreamCache: pointers.Ptr(uint64(101)), BlockSize: pointers.Ptr(uint64(102)), BufferSize: pointers.Ptr(uint64(103)), @@ -849,12 +893,6 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPVsAfterTestRun: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) { pv.Spec.MountOptions = getMountOptions(props, true, - "--streaming=true", - "--stream-cache-mb=101", - "--block-size-mb=102", - "--buffer-size-mb=103", - "--max-buffers=104", - "--max-blocks-per-file=105", "--use-adls=false") }), }, @@ -864,53 +902,6 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), } }()...) - // - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has disabled streaming", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - // Enabled: pointers.Ptr(false), - // StreamCache: pointers.Ptr(uint64(101)), - // BlockSize: pointers.Ptr(uint64(102)), - // BufferSize: pointers.Ptr(uint64(103)), - // MaxBuffers: pointers.Ptr(uint64(104)), - // MaxBlocksPerFile: pointers.Ptr(uint64(105)), - // } - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = []string{ - // "--file-cache-timeout-in-seconds=120", - // "--use-attr-cache=true", - // "--cancel-list-on-mount-seconds=0", - // "-o allow_other", - // "-o attr_timeout=120", - // "-o entry_timeout=120", - // "-o negative_timeout=120", - // "-o gid=1000", - // "--use-adls=false", - // } - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - // }()...) suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { // t.Parallel() From bfde517ca7ca31679eabf8bebae6df4aa6c6516f Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 10 Dec 2024 11:05:10 +0100 Subject: [PATCH 24/68] Correcting unit-tests --- pkg/apis/deployment/volumemount_test.go | 751 ++++++++++++------------ 1 file changed, 381 insertions(+), 370 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index d2904ec17..62bc5a5ae 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -70,6 +70,7 @@ type deploymentVolumesTestScenario struct { type pvcTestScenario struct { volumeMountTestScenario + pv corev1.PersistentVolume pvc corev1.PersistentVolumeClaim } @@ -281,35 +282,37 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { } func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { - namespace := "some-namespace" + namespace := "any-app-some-env" componentName := "some-component" + props := getPropsCsiBlobFuse2Volume1Storage1(nil) scenarios := []pvcTestScenario{ { volumeMountTestScenario: volumeMountTestScenario{ name: "Blob CSI Azure volume, PVS phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storage1", Path: "path1", GID: "1000"}, - expectedVolumeName: "csi-az-blob-some-component-volume1-storage1", + radixVolumeMount: v1.RadixVolumeMount{Name: "volume1", BlobFuse2: &v1.RadixBlobFuse2VolumeMount{Container: "storage1", GID: "1000"}, Path: "path1"}, + expectedVolumeName: "csi-blobfuse2-fuse2-some-component-volume1-storage1", expectedPvcNamePrefix: "existing-blob-pvc-name1", }, - pvc: createPvc(namespace, componentName, v1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Name = "existing-blob-pvc-name1" - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "volume1" + pvc: createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimBound }), - }, - { - volumeMountTestScenario: volumeMountTestScenario{ - name: "Blob CSI Azure volume, PVS phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume2", Storage: "storage2", Path: "path2", GID: "1000"}, - expectedVolumeName: "csi-az-blob-some-component-volume2-storage2", - expectedPvcNamePrefix: "existing-blob-pvc-name2", - }, - pvc: createPvc(namespace, componentName, v1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Name = "existing-blob-pvc-name2" - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "volume2" - pvc.Status.Phase = corev1.ClaimPending + pv: createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Status.Phase = corev1.VolumeBound }), }, + // { + // volumeMountTestScenario: volumeMountTestScenario{ + // name: "Blob CSI Azure volume, PVS phase: Pending", + // radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume2", Storage: "storage2", Path: "path2", GID: "1000"}, + // expectedVolumeName: "csi-az-blob-some-component-volume2-storage2", + // expectedPvcNamePrefix: "existing-blob-pvc-name2", + // }, + // pvc: createPvc(namespace, componentName, v1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Name = "existing-blob-pvc-name2" + // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "volume2" + // pvc.Status.Phase = corev1.ClaimPending + // }), + // }, } suite.T().Run("CSI Azure volumes with existing PVC", func(t *testing.T) { @@ -317,15 +320,23 @@ func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { testEnv := getTestEnv() for _, scenario := range scenarios { t.Logf("Scenario %s for volume mount type %s, PVC status phase '%v'", scenario.name, string(GetCsiAzureVolumeMountType(&scenario.radixVolumeMount)), scenario.pvc.Status.Phase) - _, _ = testEnv.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), &scenario.pvc, metav1.CreateOptions{}) + _, err := testEnv.kubeclient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}) + require.NoError(t, err) + _, err = testEnv.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), &scenario.pvc, metav1.CreateOptions{}) + require.NoError(t, err) + _, err = testEnv.kubeclient.CoreV1().PersistentVolumes().Create(context.Background(), &scenario.pv, metav1.CreateOptions{}) + require.NoError(t, err) component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() - volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", nil) + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", []corev1.Volume{{ + Name: props.volumeName, + VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: scenario.pvc.Name}}, + }}) assert.Nil(t, err) assert.Len(t, volumes, 1) assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name) assert.NotNil(t, volumes[0].PersistentVolumeClaim) - assert.Equal(t, volumes[0].PersistentVolumeClaim.ClaimName, scenario.pvc.Name) + assert.Equal(t, scenario.pvc.Name, volumes[0].PersistentVolumeClaim.ClaimName) } }) @@ -341,7 +352,7 @@ func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { assert.Len(t, volumes, 1) assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name) assert.NotNil(t, volumes[0].PersistentVolumeClaim) - assert.NotEqual(t, volumes[0].PersistentVolumeClaim.ClaimName, scenario.pvc.Name) + assert.NotEqual(t, scenario.pvc.Name, volumes[0].PersistentVolumeClaim.ClaimName) assert.NotContains(t, volumes[0].PersistentVolumeClaim.ClaimName, scenario.expectedPvcNamePrefix) } }) @@ -515,355 +526,355 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { environment = "some-env" componentName = "some-component" ) - // anotherNamespace := commonUtils.RandString(10) - // anotherComponentName := commonUtils.RandString(10) + anotherNamespace := commonUtils.RandString(10) + anotherComponentName := commonUtils.RandString(10) var scenarios []deploymentVolumesTestScenario - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - // }()...) - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // type scenarioProperties struct { - // changedNewRadixVolumeName string - // changedNewRadixVolumeStorageName string - // expectedVolumeName string - // expectedNewSecretName string - // expectedNewPvcName string - // expectedNewPvName string - // } - // getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Update storage in existing volume name and storage", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.Name = scenarioProps.changedNewRadixVolumeName - // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) { - // v.Name = scenarioProps.expectedVolumeName - // }), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName - // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName - // setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) - // pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - // changedNewRadixVolumeName: "volume101", - // changedNewRadixVolumeStorageName: "storage101", - // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - // expectedNewSecretName: "some-component-volume101-csiazurecreds", - // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - // expectedNewPvName: "pv-radixvolumemount-some-uuid", - // }), - // } - // }()...) - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) - // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - // pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) - // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - // matchPvAndPvc(pvForAnotherNamespace, pvcForAnotherNamespace) - // matchPvAndPvc(pvForAnotherComponent, pvcForAnotherNamespace) - // return deploymentVolumesTestScenario{ - // name: "Garbage collect orphaned PVCs and PersistentVolume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPv(props, props.namespace, props.componentName), - // pvForAnotherNamespace, - // pvForAnotherComponent, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // pvForAnotherNamespace, - // pvForAnotherComponent, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - // }()...) - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Set readonly volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPv(props, props.namespace, props.componentName), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - // }()...) - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteOnce volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // existingPvc, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - // }()...) - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteMany volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // existingPvc, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - // }()...) - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadOnlyMany volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // }), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // pv.Spec.MountOptions = getMountOptions(props, true) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - // }()...) - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - // }()...) - // scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - // StreamCache: pointers.Ptr(uint64(101)), - // BlockSize: pointers.Ptr(uint64(102)), - // BufferSize: pointers.Ptr(uint64(103)), - // MaxBuffers: pointers.Ptr(uint64(104)), - // MaxBlocksPerFile: pointers.Ptr(uint64(105)), - // } - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, - // "--streaming=true", - // "--stream-cache-mb=101", - // "--block-size-mb=102", - // "--buffer-size-mb=103", - // "--max-buffers=104", - // "--max-blocks-per-file=105", - // "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - // }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + type scenarioProperties struct { + changedNewRadixVolumeName string + changedNewRadixVolumeStorageName string + expectedVolumeName string + expectedNewSecretName string + expectedNewPvcName string + expectedNewPvName string + } + getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Update storage in existing volume name and storage", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.Name = scenarioProps.changedNewRadixVolumeName + vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) { + v.Name = scenarioProps.expectedVolumeName + }), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName + setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + changedNewRadixVolumeName: "volume101", + changedNewRadixVolumeStorageName: "storage101", + expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + expectedNewSecretName: "some-component-volume101-csiazurecreds", + expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + expectedNewPvName: "pv-radixvolumemount-some-uuid", + }), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) + pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) + pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherNamespace) + return deploymentVolumesTestScenario{ + name: "Garbage collect orphaned PVCs and PersistentVolume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createRandomPvc(props, props.namespace, props.componentName), + pvcForAnotherNamespace, + pvcForAnotherComponent, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherNamespace, + pvcForAnotherComponent, + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createRandomPv(props, props.namespace, props.componentName), + pvForAnotherNamespace, + pvForAnotherComponent, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + pvForAnotherNamespace, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Set readonly volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createRandomPvc(props, props.namespace, props.componentName), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createRandomPv(props, props.namespace, props.componentName), + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteOnce volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteMany volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadOnlyMany volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + pv.Spec.MountOptions = getMountOptions(props, true) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, + "--streaming=true", + "--stream-cache-mb=101", + "--block-size-mb=102", + "--buffer-size-mb=103", + "--max-buffers=104", + "--max-blocks-per-file=105", + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { @@ -904,7 +915,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }()...) suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { - // t.Parallel() + t.Parallel() for _, factory := range suite.radixCommonDeployComponentFactories[:1] { for _, scenario := range scenarios { t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) From b2bcb854e5bf60aa133e3e6d1cd6519ce86cbf72 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 10 Dec 2024 11:32:38 +0100 Subject: [PATCH 25/68] Correcting unit-tests --- pkg/apis/deployment/volumemount_test.go | 36 +++++++------------------ 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 62bc5a5ae..c37423987 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -288,38 +288,20 @@ func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { scenarios := []pvcTestScenario{ { volumeMountTestScenario: volumeMountTestScenario{ - name: "Blob CSI Azure volume, PVS phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Name: "volume1", BlobFuse2: &v1.RadixBlobFuse2VolumeMount{Container: "storage1", GID: "1000"}, Path: "path1"}, - expectedVolumeName: "csi-blobfuse2-fuse2-some-component-volume1-storage1", - expectedPvcNamePrefix: "existing-blob-pvc-name1", + name: "Blob CSI Azure BlobFuse2 Fuse2 volume", + radixVolumeMount: v1.RadixVolumeMount{Name: "volume1", BlobFuse2: &v1.RadixBlobFuse2VolumeMount{Container: "storage1", GID: "1000"}, Path: "path1"}, + expectedVolumeName: "csi-blobfuse2-fuse2-some-component-volume1-storage1", }, - pvc: createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Status.Phase = corev1.ClaimBound - }), - pv: createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Status.Phase = corev1.VolumeBound - }), + pvc: createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pv: createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), }, - // { - // volumeMountTestScenario: volumeMountTestScenario{ - // name: "Blob CSI Azure volume, PVS phase: Pending", - // radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume2", Storage: "storage2", Path: "path2", GID: "1000"}, - // expectedVolumeName: "csi-az-blob-some-component-volume2-storage2", - // expectedPvcNamePrefix: "existing-blob-pvc-name2", - // }, - // pvc: createPvc(namespace, componentName, v1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Name = "existing-blob-pvc-name2" - // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = "volume2" - // pvc.Status.Phase = corev1.ClaimPending - // }), - // }, } suite.T().Run("CSI Azure volumes with existing PVC", func(t *testing.T) { t.Parallel() - testEnv := getTestEnv() for _, scenario := range scenarios { t.Logf("Scenario %s for volume mount type %s, PVC status phase '%v'", scenario.name, string(GetCsiAzureVolumeMountType(&scenario.radixVolumeMount)), scenario.pvc.Status.Phase) + testEnv := getTestEnv() _, err := testEnv.kubeclient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}) require.NoError(t, err) _, err = testEnv.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), &scenario.pvc, metav1.CreateOptions{}) @@ -342,10 +324,11 @@ func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { suite.T().Run("CSI Azure volumes with no existing PVC", func(t *testing.T) { t.Parallel() - testEnv := getTestEnv() for _, scenario := range scenarios { t.Logf("Scenario %s for volume mount type %s, PVC status phase '%v'", scenario.name, string(GetCsiAzureVolumeMountType(&scenario.radixVolumeMount)), scenario.pvc.Status.Phase) - + testEnv := getTestEnv() + _, err := testEnv.kubeclient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}) + require.NoError(t, err) component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(scenario.radixVolumeMount).BuildComponent() volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, &component, "", nil) assert.Nil(t, err) @@ -353,7 +336,6 @@ func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { assert.Equal(t, scenario.expectedVolumeName, volumes[0].Name) assert.NotNil(t, volumes[0].PersistentVolumeClaim) assert.NotEqual(t, scenario.pvc.Name, volumes[0].PersistentVolumeClaim.ClaimName) - assert.NotContains(t, volumes[0].PersistentVolumeClaim.ClaimName, scenario.expectedPvcNamePrefix) } }) } From a6d960e69f1b226d4886dc30f7124c1fe248b893 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 12 Dec 2024 16:29:38 +0100 Subject: [PATCH 26/68] Fixing unit-tests --- pkg/apis/batch/syncer_test.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/apis/batch/syncer_test.go b/pkg/apis/batch/syncer_test.go index deb82d07a..7cf3e92f2 100644 --- a/pkg/apis/batch/syncer_test.go +++ b/pkg/apis/batch/syncer_test.go @@ -1084,8 +1084,8 @@ func (s *syncerTestSuite) Test_JobWithVolumeMounts() { jobs, _ := s.kubeClient.BatchV1().Jobs(namespace).List(context.Background(), metav1.ListOptions{}) s.Require().Len(jobs.Items, 1) job := slice.FindAll(jobs.Items, func(job batchv1.Job) bool { return job.GetName() == getKubeJobName(batchName, jobName) })[0] - s.Require().Len(job.Spec.Template.Spec.Volumes, 2) - s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 2) + s.Require().Len(job.Spec.Template.Spec.Volumes, 1) + s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 1) s.Equal(job.Spec.Template.Spec.Volumes[0].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name) s.Equal("/azureblob2path", job.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath) } @@ -1111,7 +1111,6 @@ func (s *syncerTestSuite) Test_JobWithVolumeMounts_Deprecated() { { Name: componentName, VolumeMounts: []radixv1.RadixVolumeMount{ - {Type: "blob", Name: "blobname", Container: "blobcontainer", Path: "/blobpath"}, {Type: "azure-blob", Name: "azureblobname", Storage: "azureblobcontainer", Path: "/azureblobpath"}, }, }, @@ -1128,12 +1127,10 @@ func (s *syncerTestSuite) Test_JobWithVolumeMounts_Deprecated() { jobs, _ := s.kubeClient.BatchV1().Jobs(namespace).List(context.Background(), metav1.ListOptions{}) s.Require().Len(jobs.Items, 1) job := slice.FindAll(jobs.Items, func(job batchv1.Job) bool { return job.GetName() == getKubeJobName(batchName, jobName) })[0] - s.Require().Len(job.Spec.Template.Spec.Volumes, 2) - s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 2) + s.Require().Len(job.Spec.Template.Spec.Volumes, 1) + s.Require().Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 1) s.Equal(job.Spec.Template.Spec.Volumes[0].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name) - s.Equal(job.Spec.Template.Spec.Volumes[1].Name, job.Spec.Template.Spec.Containers[0].VolumeMounts[1].Name) - s.Equal("/blobpath", job.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath) - s.Equal("/azureblobpath", job.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath) + s.Equal("/azureblobpath", job.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath) } func (s *syncerTestSuite) Test_JobWithAzureSecretRefs() { From 88c224611af6a01d83ab5d417769f904d8495ef2 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 12 Dec 2024 16:58:30 +0100 Subject: [PATCH 27/68] Updated ref --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index c900f234b..ebe0da8b0 100644 --- a/go.mod +++ b/go.mod @@ -21,9 +21,9 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - golang.org/x/crypto v0.26.0 + golang.org/x/crypto v0.31.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/sync v0.8.0 + golang.org/x/sync v0.10.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.31.0 k8s.io/apiextensions-apiserver v0.31.0 @@ -84,9 +84,9 @@ require ( golang.org/x/mod v0.20.0 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 06838d4bf..cdb22974e 100644 --- a/go.sum +++ b/go.sum @@ -177,8 +177,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -199,8 +199,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -210,15 +210,15 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From b79f6bcd7ccc44777cebd4610cb61201fcc06740 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 13 Dec 2024 12:21:26 +0100 Subject: [PATCH 28/68] Updated ref --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ebe0da8b0..be25e4890 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.5 require ( dario.cat/mergo v1.0.1 github.com/cert-manager/cert-manager v1.15.4 - github.com/equinor/radix-common v1.9.4 + github.com/equinor/radix-common v1.9.5 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index cdb22974e..f7fc93e93 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/equinor/radix-common v1.9.4 h1:ErSnB2tqlRwaQuQdaA0qzsReDtHDgubcvqRO098ncEw= -github.com/equinor/radix-common v1.9.4/go.mod h1:+g0Wj0D40zz29DjNkYKVmCVeYy4OsFWKI7Qi9rA6kpY= +github.com/equinor/radix-common v1.9.5 h1:p1xldkYUoavwIMguoxxOyVkOXLPA6K8qMsgzeztQtQw= +github.com/equinor/radix-common v1.9.5/go.mod h1:+g0Wj0D40zz29DjNkYKVmCVeYy4OsFWKI7Qi9rA6kpY= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= From 124570b2c71d77b55e1e07b7449ba509dca91373 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 13 Dec 2024 13:21:32 +0100 Subject: [PATCH 29/68] Fixed unit tests --- pkg/apis/deployment/deployment_test.go | 18 ++--- .../radixvalidators/testdata/radixconfig.yaml | 4 +- pkg/apis/radixvalidators/validate_ra.go | 4 +- pkg/apis/radixvalidators/validate_ra_test.go | 72 ++----------------- 4 files changed, 14 insertions(+), 84 deletions(-) diff --git a/pkg/apis/deployment/deployment_test.go b/pkg/apis/deployment/deployment_test.go index daaea7631..851178604 100644 --- a/pkg/apis/deployment/deployment_test.go +++ b/pkg/apis/deployment/deployment_test.go @@ -130,7 +130,6 @@ func TestObjectSynced_MultiComponent_ContainsAllElements(t *testing.T) { outdatedSecret := "outdatedSecret" remainingSecret := "remainingSecret" addingSecret := "addingSecret" - blobVolumeName := "blob_volume_1" blobCsiAzureVolumeName := "blobCsiAzure_volume_1" if componentsExist { @@ -324,8 +323,8 @@ func TestObjectSynced_MultiComponent_ContainsAllElements(t *testing.T) { assert.True(t, envVariableByNameExistOnDeployment(addingSecret, componentNameRadixQuote, deployments)) } - volumesExist := len(spec.Template.Spec.Volumes) > 1 - volumeMountsExist := len(spec.Template.Spec.Containers[0].VolumeMounts) > 1 + volumesExist := len(spec.Template.Spec.Volumes) > 0 + volumeMountsExist := len(spec.Template.Spec.Containers[0].VolumeMounts) > 0 if !componentsExist { assert.True(t, volumesExist, "expected existing volumes") assert.True(t, volumeMountsExist, "expected existing volume mounts") @@ -388,20 +387,18 @@ func TestObjectSynced_MultiComponent_ContainsAllElements(t *testing.T) { secrets, _ := kubeclient.CoreV1().Secrets(envNamespace).List(context.Background(), metav1.ListOptions{}) if !componentsExist { - assert.Equal(t, 3, len(secrets.Items), "Number of secrets was not according to spec") + assert.Equal(t, 2, len(secrets.Items), "Number of secrets was not according to spec") } else { assert.Equal(t, 1, len(secrets.Items), "Number of secrets was not according to spec") } componentSecretName := utils.GetComponentSecretName(componentNameRadixQuote) assert.True(t, secretByNameExists(componentSecretName, secrets), "Component secret is not as expected") - blobFuseSecretExists := secretByNameExists(defaults.GetBlobFuseCredsSecretName(componentNameRadixQuote, blobVolumeName), secrets) blobCsiAzureFuseSecretExists := secretByNameExists(defaults.GetCsiAzureVolumeMountCredsSecretName(componentNameRadixQuote, blobCsiAzureVolumeName), secrets) if !componentsExist { - assert.True(t, blobFuseSecretExists, "expected Blobfuse volume mount secret") assert.True(t, blobCsiAzureFuseSecretExists, "expected blob CSI Azure volume mount secret") } else { - assert.False(t, blobFuseSecretExists, "unexpected volume mount secrets") + assert.False(t, blobCsiAzureFuseSecretExists, "unexpected volume mount secrets") } }) @@ -522,7 +519,6 @@ func TestObjectSynced_MultiJob_ContainsAllElements(t *testing.T) { outdatedSecret := "outdatedSecret" remainingSecret := "remainingSecret" addingSecret := "addingSecret" - blobVolumeName := "blob_volume_1" blobCsiAzureVolumeName := "blobCsiAzure_volume_1" payloadPath := "payloadpath" if jobsExist { @@ -711,7 +707,7 @@ func TestObjectSynced_MultiJob_ContainsAllElements(t *testing.T) { secrets, _ := kubeclient.CoreV1().Secrets(envNamespace).List(context.Background(), metav1.ListOptions{}) if !jobsExist { - assert.Equal(t, 3, len(secrets.Items), "Number of secrets was not according to spec") + assert.Equal(t, 2, len(secrets.Items), "Number of secrets was not according to spec") } else { assert.Equal(t, 1, len(secrets.Items), "Number of secrets was not according to spec") } @@ -719,13 +715,11 @@ func TestObjectSynced_MultiJob_ContainsAllElements(t *testing.T) { jobSecretName := utils.GetComponentSecretName(jobName) assert.True(t, secretByNameExists(jobSecretName, secrets), "Job secret is not as expected") - blobFuseSecretExists := secretByNameExists(defaults.GetBlobFuseCredsSecretName(jobName, blobVolumeName), secrets) blobCsiAzureFuseSecretExists := secretByNameExists(defaults.GetCsiAzureVolumeMountCredsSecretName(jobName, blobCsiAzureVolumeName), secrets) if !jobsExist { - assert.True(t, blobFuseSecretExists, "expected Blobfuse volume mount secret") assert.True(t, blobCsiAzureFuseSecretExists, "expected blob CSI Azure volume mount secret") } else { - assert.False(t, blobFuseSecretExists, "unexpected volume mount secrets") + assert.False(t, blobCsiAzureFuseSecretExists, "unexpected volume mount secrets") } }) diff --git a/pkg/apis/radixvalidators/testdata/radixconfig.yaml b/pkg/apis/radixvalidators/testdata/radixconfig.yaml index 5a65b9b71..c649b6385 100644 --- a/pkg/apis/radixvalidators/testdata/radixconfig.yaml +++ b/pkg/apis/radixvalidators/testdata/radixconfig.yaml @@ -93,7 +93,7 @@ spec: azure: clientId: 11111111-2222-3333-4444-555555555555 volumeMounts: - - type: blob + - type: azure-blob name: blobvol container: blobcontainer path: /path/to/mount @@ -202,7 +202,7 @@ spec: azure: clientId: 11111111-2222-3333-4444-555555555555 volumeMounts: - - type: blob + - type: azure-blob name: blobvol container: blobcontainer path: /path/to/mount diff --git a/pkg/apis/radixvalidators/validate_ra.go b/pkg/apis/radixvalidators/validate_ra.go index 31de0eb88..31b9b6aca 100644 --- a/pkg/apis/radixvalidators/validate_ra.go +++ b/pkg/apis/radixvalidators/validate_ra.go @@ -1622,8 +1622,8 @@ func validateVolumeMountDeprecatedSource(v *radixv1.RadixVolumeMount) error { return volumeMountDeprecatedSourceValidationError(fmt.Errorf("%w. %w", ErrVolumeMountInvalidRequestsStorage, err)) } } - if v.Type == radixv1.MountTypeBlobFuse2FuseCsiAzure && len(v.Storage) == 0 { - return volumeMountDeprecatedSourceValidationError(ErrVolumeMountMissingStorage) + if v.Type == radixv1.MountTypeBlobFuse2FuseCsiAzure && len(v.Container) == 0 { + return volumeMountBlobFuse2ValidationError(ErrVolumeMountMissingContainer) } return nil } diff --git a/pkg/apis/radixvalidators/validate_ra_test.go b/pkg/apis/radixvalidators/validate_ra_test.go index 064afcb1e..d994da3a0 100644 --- a/pkg/apis/radixvalidators/validate_ra_test.go +++ b/pkg/apis/radixvalidators/validate_ra_test.go @@ -1334,46 +1334,14 @@ func Test_ValidationOfVolumeMounts_Errors(t *testing.T) { updateRA: setComponentAndJobsVolumeMounts, expectedError: radixvalidators.ErrVolumeMountMissingType, }, - "deprecated blob: valid": { + "deprecated azure-blob: valid": { volumeMounts: func() []radixv1.RadixVolumeMount { volumeMounts := []radixv1.RadixVolumeMount{ { - Type: "blob", + Type: "azure-blob", Name: "some_name", Path: "some_path", - Container: "any-container", - // RequestsStorage: "50M", - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: nil, - }, - "deprecated blob: missing container": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Type: "blob", - Name: "some_name", - Path: "some_path", - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountMissingContainer, - }, - "deprecated azure-blob: valid": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Type: "azure-blob", - Name: "some_name", - Path: "some_path", - Storage: "any-storage", + Container: "any-storage", }, } @@ -1395,23 +1363,7 @@ func Test_ValidationOfVolumeMounts_Errors(t *testing.T) { return volumeMounts }, updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountMissingStorage, - }, - "deprecated azure-file: valid": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Type: "azure-file", - Name: "some_name", - Path: "some_path", - Storage: "any-storage", - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: nil, + expectedError: radixvalidators.ErrVolumeMountMissingContainer, }, "deprecated common: invalid type": { volumeMounts: func() []radixv1.RadixVolumeMount { @@ -1428,22 +1380,6 @@ func Test_ValidationOfVolumeMounts_Errors(t *testing.T) { updateRA: setComponentAndJobsVolumeMounts, expectedError: radixvalidators.ErrVolumeMountInvalidType, }, - "deprecated common: invalid requestsStorage": { - volumeMounts: func() []radixv1.RadixVolumeMount { - volumeMounts := []radixv1.RadixVolumeMount{ - { - Type: "blob", - Name: "some_name", - Path: "some_path", - RequestsStorage: "50x", - }, - } - - return volumeMounts - }, - updateRA: setComponentAndJobsVolumeMounts, - expectedError: radixvalidators.ErrVolumeMountInvalidRequestsStorage, - }, "blobfuse2: valid": { volumeMounts: func() []radixv1.RadixVolumeMount { volumeMounts := []radixv1.RadixVolumeMount{ From a7d09df087747f6f7a9f5ca998c025fe67a8bdbf Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 13 Dec 2024 15:54:14 +0100 Subject: [PATCH 30/68] Fixed unit tests --- pkg/apis/deployment/volumemount_test.go | 891 +++++++++++------- .../internal/persistentvolume/defaults.go | 13 +- .../persistentvolume/persistentvolume.go | 17 +- 3 files changed, 556 insertions(+), 365 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index c37423987..2b14ddea0 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -4,8 +4,11 @@ import ( "context" "encoding/json" "fmt" + "github.com/google/uuid" + "k8s.io/apimachinery/pkg/types" "strings" "testing" + "time" commonUtils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/pointers" @@ -508,14 +511,401 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { environment = "some-env" componentName = "some-component" ) - anotherNamespace := commonUtils.RandString(10) + //anotherNamespace := commonUtils.RandString(10) anotherComponentName := commonUtils.RandString(10) var scenarios []deploymentVolumesTestScenario + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // type scenarioProperties struct { + // changedNewRadixVolumeName string + // changedNewRadixVolumeStorageName string + // expectedVolumeName string + // expectedNewSecretName string + // expectedNewPvcName string + // expectedNewPvName string + // } + // getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Update storage in existing volume name and storage", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.Name = scenarioProps.changedNewRadixVolumeName + // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) { + // v.Name = scenarioProps.expectedVolumeName + // }), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName + // setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + // pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + // changedNewRadixVolumeName: "volume101", + // changedNewRadixVolumeStorageName: "storage101", + // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + // expectedNewSecretName: "some-component-volume101-csiazurecreds", + // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + // expectedNewPvName: "pv-radixvolumemount-some-uuid", + // }), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) + // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + // pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) + // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + // matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // return deploymentVolumesTestScenario{ + // name: "Garbage collect orphaned PVCs and PersistentVolume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPv(props, props.namespace, props.componentName), + // pvForAnotherNamespace, + // pvForAnotherComponent, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // pvForAnotherNamespace, + // pvForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Set readonly volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPv(props, props.namespace, props.componentName), + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteOnce volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // existingPvc, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteMany volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // existingPvc, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadOnlyMany volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // }), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // pv.Spec.MountOptions = getMountOptions(props, true) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + // StreamCache: pointers.Ptr(uint64(101)), + // BlockSize: pointers.Ptr(uint64(102)), + // BufferSize: pointers.Ptr(uint64(103)), + // MaxBuffers: pointers.Ptr(uint64(104)), + // MaxBlocksPerFile: pointers.Ptr(uint64(105)), + // } + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, + // "--streaming=true", + // "--stream-cache-mb=101", + // "--block-size-mb=102", + // "--buffer-size-mb=103", + // "--max-buffers=104", + // "--max-blocks-per-file=105", + // "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) + // + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has disabled streaming", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + // Enabled: pointers.Ptr(false), + // StreamCache: pointers.Ptr(uint64(101)), + // BlockSize: pointers.Ptr(uint64(102)), + // BufferSize: pointers.Ptr(uint64(103)), + // MaxBuffers: pointers.Ptr(uint64(104)), + // MaxBlocksPerFile: pointers.Ptr(uint64(105)), + // } + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, + // "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) + pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) return deploymentVolumesTestScenario{ - name: "Create new volume", + name: "Do not change existing PersistentVolume, when creating new PVC", props: props, radixVolumeMounts: []v1.RadixVolumeMount{ createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), @@ -523,264 +913,20 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - type scenarioProperties struct { - changedNewRadixVolumeName string - changedNewRadixVolumeStorageName string - expectedVolumeName string - expectedNewSecretName string - expectedNewPvcName string - expectedNewPvName string - } - getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Update storage in existing volume name and storage", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.Name = scenarioProps.changedNewRadixVolumeName - vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) { - v.Name = scenarioProps.expectedVolumeName - }), - }, existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.ObjectMeta.Name = scenarioProps.expectedNewPvName - pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName - setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) - pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - changedNewRadixVolumeName: "volume101", - changedNewRadixVolumeStorageName: "storage101", - expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - expectedNewSecretName: "some-component-volume101-csiazurecreds", - expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - expectedNewPvName: "pv-radixvolumemount-some-uuid", - }), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) - pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) - pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) - matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherNamespace) - return deploymentVolumesTestScenario{ - name: "Garbage collect orphaned PVCs and PersistentVolume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - pvcForAnotherNamespace, pvcForAnotherComponent, }, existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - pvcForAnotherNamespace, pvcForAnotherComponent, }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - createRandomPv(props, props.namespace, props.componentName), - pvForAnotherNamespace, - pvForAnotherComponent, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - pvForAnotherNamespace, - pvForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Set readonly volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - createRandomPv(props, props.namespace, props.componentName), - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadWriteOnce volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - existingPvc, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - }), - }, existingPVsBeforeTestRun: []corev1.PersistentVolume{ existingPv, + pvForAnotherComponent, }, existingPVsAfterTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadWriteMany volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - existingPvc, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ existingPv, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadOnlyMany volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - }), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - pv.Spec.MountOptions = getMountOptions(props, true) - }), + pvForAnotherComponent, }, } } @@ -788,116 +934,9 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { getScenario(getPropsCsiBlobVolume1Storage1(nil)), } }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, - "--streaming=true", - "--stream-cache-mb=101", - "--block-size-mb=102", - "--buffer-size-mb=103", - "--max-buffers=104", - "--max-blocks-per-file=105", - "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has disabled streaming", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - Enabled: pointers.Ptr(false), - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, - "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { - t.Parallel() + //t.Parallel() for _, factory := range suite.radixCommonDeployComponentFactories[:1] { for _, scenario := range scenarios { t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) @@ -907,6 +946,9 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { putExistingDeploymentVolumesScenarioDataToFakeCluster(&scenario, deployment) desiredDeployment := getDesiredDeployment(componentName, scenario.volumes) + //existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment) + //assert.Nil(t, err) + // // action deployComponent := deployment.radixDeployment.Spec.Components[0] err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), &deployComponent, desiredDeployment) @@ -1163,6 +1205,28 @@ func createRandomPvc(props expectedPvcPvProperties, namespace, componentName str }) } +func createRandomAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolume { + return createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) { + pvName := "pvc-" + uuid.NewString() + pv.ObjectMeta.Name = pvName + pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvName] = pvName + }) +} + +func createRandomAutoProvisionedPvcWithStorageClass(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolumeClaim { + return createExpectedAutoProvisionedPvcWithStorageClass(props, func(pvc *corev1.PersistentVolumeClaim) { + pvcName, err := getCsiAzurePersistentVolumeClaimName(componentName, &v1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: v1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.radixStorageName, Path: "/tmp"}) + if err != nil { + panic(err) + } + pvName := getCsiAzurePersistentVolumeName() + pvc.ObjectMeta.Name = pvcName + pvc.ObjectMeta.Namespace = namespace + pvc.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName + pvc.Spec.VolumeName = pvName + }) +} + func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcPvProperties)) expectedPvcPvProperties { appName := "any-app" environment := "some-env" @@ -1367,6 +1431,58 @@ func createExpectedPv(props expectedPvcPvProperties, modify func(pv *corev1.Pers return pv } +func createAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties, modify func(pv *corev1.PersistentVolume)) corev1.PersistentVolume { + mountOptions := getMountOptionsInRandomOrder(props, true) + pv := corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: props.persistentVolumeName, + Annotations: map[string]string{ + persistentvolume.CsiAnnotationProvisionedBy: provisionerBlobCsiAzure, + persistentvolume.CsiAnnotationProvisionerDeletionSecretName: props.pvSecretName, + persistentvolume.CsiAnnotationProvisionerDeletionSecretNamespace: props.namespace, + }, + }, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: provisionerBlobCsiAzure, + VolumeHandle: "MC_clusters_ABC_northeurope##testdata#pvc-681b9ffc-66cc-4e09-90b2-872688b792542#some-app-namespace#", + VolumeAttributes: map[string]string{ + persistentvolume.CsiVolumeMountAttributeContainerName: props.radixStorageName, + persistentvolume.CsiVolumeMountAttributeProtocol: persistentvolume.CsiVolumeAttributeProtocolParameterFuse2, + persistentvolume.CsiVolumeMountAttributePvName: props.persistentVolumeName, + persistentvolume.CsiVolumeMountAttributePvcName: props.pvcName, + persistentvolume.CsiVolumeMountAttributePvcNamespace: props.namespace, + persistentvolume.CsiVolumeMountAttributeSecretNamespace: props.namespace, + persistentvolume.CsiVolumeMountAttributeProvisionerIdentity: "6540128941979-5154-blob.csi.azure.com", + }, + NodeStageSecretRef: &corev1.SecretReference{ + Name: props.pvSecretName, + Namespace: props.namespace, + }, + }, + }, + AccessModes: []corev1.PersistentVolumeAccessMode{props.volumeAccessMode}, + ClaimRef: &corev1.ObjectReference{ + Namespace: props.namespace, + Name: props.pvcName, + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + StorageClassName: "some-storage-class", + MountOptions: mountOptions, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + }, + Status: corev1.PersistentVolumeStatus{Phase: corev1.VolumeBound, LastPhaseTransitionTime: pointers.Ptr(metav1.Time{Time: time.Now()})}, + } + setVolumeMountAttribute(&pv, props.radixVolumeMountType, props.radixStorageName, props.pvcName) + if modify != nil { + modify(&pv) + } + return pv +} + func getMountOptions(props expectedPvcPvProperties, readOnly bool, extraOptions ...string) []string { options := []string{ "--file-cache-timeout-in-seconds=120", @@ -1387,6 +1503,26 @@ func getMountOptions(props expectedPvcPvProperties, readOnly bool, extraOptions return append(options, extraOptions...) } +func getMountOptionsInRandomOrder(props expectedPvcPvProperties, readOnly bool, extraOptions ...string) []string { + options := []string{ + "--file-cache-timeout-in-seconds=120", + "--use-attr-cache=true", + "-o allow_other", + "--cancel-list-on-mount-seconds=0", + "-o negative_timeout=120", + "-o entry_timeout=120", + "-o attr_timeout=120", + } + idOption := getPersistentVolumeIdMountOption(props) + if len(idOption) > 0 { + options = append(options, idOption) + } + if readOnly { + options = append(options, "-o ro") + } + return append(options, extraOptions...) +} + func setVolumeMountAttribute(pv *corev1.PersistentVolume, radixVolumeMountType v1.MountType, containerName, pvcName string) { pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributeContainerName] = containerName pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvcName @@ -1436,6 +1572,49 @@ func createExpectedPvc(props expectedPvcPvProperties, modify func(*corev1.Persis return pvc } +func createExpectedAutoProvisionedPvcWithStorageClass(props expectedPvcPvProperties, modify func(*corev1.PersistentVolumeClaim)) corev1.PersistentVolumeClaim { + labels := map[string]string{ + kube.RadixAppLabel: props.appName, + kube.RadixComponentLabel: props.componentName, + kube.RadixMountTypeLabel: string(props.radixVolumeMountType), + kube.RadixVolumeMountNameLabel: props.radixVolumeMountName, + } + annotations := map[string]string{ + "pv.kubernetes.io/bind-completed": "yes", + "pv.kubernetes.io/bound-by-controller": "yes", + "volume.beta.kubernetes.io/storage-provisioner": "blob.csi.azure.com", + "volume.kubernetes.io/storage-provisioner": "blob.csi.azure.com", + } + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: props.pvcName, + Namespace: props.namespace, + Labels: labels, + Annotations: annotations, + Finalizers: []string{"kubernetes.io/pvc-protection"}, + ResourceVersion: "630363277", + UID: types.UID("681b9ffc-66cc-4e09-90b2-872688b792542"), + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{props.volumeAccessMode}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, // it seems correct number is not needed for CSI driver + }, + VolumeName: props.persistentVolumeName, + StorageClassName: pointers.Ptr("some-storage-class"), + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: corev1.ClaimBound, + AccessModes: []corev1.PersistentVolumeAccessMode{props.volumeAccessMode}, + Capacity: map[corev1.ResourceName]resource.Quantity{corev1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, + }, + } + if modify != nil { + modify(&pvc) + } + return pvc +} + func createTestVolume(pvcProps expectedPvcPvProperties, modify func(*corev1.Volume)) corev1.Volume { volume := corev1.Volume{ Name: pvcProps.volumeName, diff --git a/pkg/apis/internal/persistentvolume/defaults.go b/pkg/apis/internal/persistentvolume/defaults.go index c65849553..05a8591b3 100644 --- a/pkg/apis/internal/persistentvolume/defaults.go +++ b/pkg/apis/internal/persistentvolume/defaults.go @@ -27,10 +27,11 @@ const ( CsiVolumeSourceDriverSecretStore = "secrets-store.csi.k8s.io" CsiVolumeSourceVolumeAttributeSecretProviderClass = "secretProviderClass" - CsiVolumeMountAttributePvName = "csi.storage.k8s.io/pv/name" - CsiVolumeMountAttributePvcName = "csi.storage.k8s.io/pvc/name" - CsiVolumeMountAttributePvcNamespace = "csi.storage.k8s.io/pvc/namespace" - CsiVolumeMountAttributeSecretNamespace = "secretnamespace" - CsiVolumeMountAttributeProtocol = "protocol" // Protocol - CsiVolumeMountAttributeContainerName = "containerName" // Container name - foc container storages + CsiVolumeMountAttributePvName = "csi.storage.k8s.io/pv/name" + CsiVolumeMountAttributePvcName = "csi.storage.k8s.io/pvc/name" + CsiVolumeMountAttributePvcNamespace = "csi.storage.k8s.io/pvc/namespace" + CsiVolumeMountAttributeSecretNamespace = "secretnamespace" + CsiVolumeMountAttributeProtocol = "protocol" // Protocol + CsiVolumeMountAttributeContainerName = "containerName" // Container name - foc container storages + CsiVolumeMountAttributeProvisionerIdentity = "storage.kubernetes.io/csiProvisionerIdentity" ) diff --git a/pkg/apis/internal/persistentvolume/persistentvolume.go b/pkg/apis/internal/persistentvolume/persistentvolume.go index f14e75689..11374c689 100644 --- a/pkg/apis/internal/persistentvolume/persistentvolume.go +++ b/pkg/apis/internal/persistentvolume/persistentvolume.go @@ -20,7 +20,7 @@ func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { if !utils.EqualStringMaps(getPvAnnotations(pv1), getPvAnnotations(pv2)) { return false } - if !utils.EqualStringMaps(pv1.Spec.CSI.VolumeAttributes, pv2.Spec.CSI.VolumeAttributes) { + if !utils.EqualStringMaps(getVolumeAttributes(pv1), getVolumeAttributes(pv2)) { return false } if !utils.EqualStringMaps(getMountOptionsMap(pv1.Spec.MountOptions), getMountOptionsMap(pv2.Spec.MountOptions)) { @@ -62,8 +62,8 @@ func EqualPersistentVolumesForTest(expectedPv, actualPv *corev1.PersistentVolume if !utils.EqualStringMaps(getPvAnnotations(expectedPv), getPvAnnotations(actualPv)) { return false } - expectedClonedAttrs := cloneMap(expectedPv.Spec.CSI.VolumeAttributes, CsiVolumeMountAttributePvName, CsiVolumeMountAttributePvcName) - actualClonedAttrs := cloneMap(actualPv.Spec.CSI.VolumeAttributes, CsiVolumeMountAttributePvName, CsiVolumeMountAttributePvcName) + expectedClonedAttrs := cloneMap(expectedPv.Spec.CSI.VolumeAttributes, CsiVolumeMountAttributePvName, CsiVolumeMountAttributePvcName, CsiVolumeMountAttributeProvisionerIdentity) + actualClonedAttrs := cloneMap(actualPv.Spec.CSI.VolumeAttributes, CsiVolumeMountAttributePvName, CsiVolumeMountAttributePvcName, CsiVolumeMountAttributeProvisionerIdentity) if !utils.EqualStringMaps(expectedClonedAttrs, actualClonedAttrs) { return false } @@ -137,6 +137,17 @@ func getPvAnnotations(pv *corev1.PersistentVolume) map[string]string { return annotations } +func getVolumeAttributes(pv *corev1.PersistentVolume) map[string]string { + attributes := make(map[string]string) + for key, value := range pv.Spec.CSI.VolumeAttributes { + if key == CsiVolumeMountAttributeProvisionerIdentity { + continue // ignore automatically added attribute(s) + } + attributes[key] = value + } + return attributes +} + func getMountOptionsMap(mountOptions []string) map[string]string { return slice.Reduce(mountOptions, make(map[string]string), func(acc map[string]string, item string) map[string]string { if len(item) == 0 { From f2ce6bea72a99b09b8a8ee39c27e453c96748753 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 13 Dec 2024 16:21:46 +0100 Subject: [PATCH 31/68] Fixed unit tests --- pkg/apis/deployment/volumemount_test.go | 44 ++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 2b14ddea0..5d51a46e2 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -898,14 +898,50 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), // } //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) + // pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) + // return deploymentVolumesTestScenario{ + // name: "Do not change existing PersistentVolume with class name, when creating new PVC", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // pvcForAnotherComponent, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherComponent, + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) - pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) + pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) + existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) return deploymentVolumesTestScenario{ - name: "Do not change existing PersistentVolume, when creating new PVC", + name: "Do not change existing PersistentVolume without class name, when creating new PVC", props: props, radixVolumeMounts: []v1.RadixVolumeMount{ createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), From 5554640e7d87b02e4475abb33de6f0c468670fa5 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 16 Dec 2024 10:03:19 +0100 Subject: [PATCH 32/68] Added yq install --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index d11c38f83..a10f6f46c 100644 --- a/Makefile +++ b/Makefile @@ -173,6 +173,7 @@ verify-generate: bootstrap generate HAS_GOLANGCI_LINT := $(shell command -v golangci-lint;) HAS_MOCKGEN := $(shell command -v mockgen;) HAS_CONTROLLER_GEN := $(shell command -v controller-gen;) +HAS_YQ := $(shell command -v yq;) .PHONY: bootstrap bootstrap: vendor @@ -185,3 +186,6 @@ endif ifndef HAS_CONTROLLER_GEN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.2 endif +ifndef HAS_YQ + go install github.com/mikefarah/yq/v4@latest +endif From 1fc3ac67abfcf1407c45b4faeb3373b7b8d47399 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 16 Dec 2024 11:15:18 +0100 Subject: [PATCH 33/68] Restored tests --- pkg/apis/deployment/volumemount_test.go | 840 ++++++++++++------------ 1 file changed, 420 insertions(+), 420 deletions(-) diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index 5d51a46e2..cd7885e4f 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -511,429 +511,429 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { environment = "some-env" componentName = "some-component" ) - //anotherNamespace := commonUtils.RandString(10) + anotherNamespace := commonUtils.RandString(10) anotherComponentName := commonUtils.RandString(10) var scenarios []deploymentVolumesTestScenario - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // type scenarioProperties struct { - // changedNewRadixVolumeName string - // changedNewRadixVolumeStorageName string - // expectedVolumeName string - // expectedNewSecretName string - // expectedNewPvcName string - // expectedNewPvName string - // } - // getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Update storage in existing volume name and storage", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.Name = scenarioProps.changedNewRadixVolumeName - // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) { - // v.Name = scenarioProps.expectedVolumeName - // }), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName - // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName - // setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) - // pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - // changedNewRadixVolumeName: "volume101", - // changedNewRadixVolumeStorageName: "storage101", - // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - // expectedNewSecretName: "some-component-volume101-csiazurecreds", - // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - // expectedNewPvName: "pv-radixvolumemount-some-uuid", - // }), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) - // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - // pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) - // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - // matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // return deploymentVolumesTestScenario{ - // name: "Garbage collect orphaned PVCs and PersistentVolume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPv(props, props.namespace, props.componentName), - // pvForAnotherNamespace, - // pvForAnotherComponent, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // pvForAnotherNamespace, - // pvForAnotherComponent, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Set readonly volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPv(props, props.namespace, props.componentName), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteOnce volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // existingPvc, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteMany volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // existingPvc, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadOnlyMany volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // }), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // pv.Spec.MountOptions = getMountOptions(props, true) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - // StreamCache: pointers.Ptr(uint64(101)), - // BlockSize: pointers.Ptr(uint64(102)), - // BufferSize: pointers.Ptr(uint64(103)), - // MaxBuffers: pointers.Ptr(uint64(104)), - // MaxBlocksPerFile: pointers.Ptr(uint64(105)), - // } - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, - // "--streaming=true", - // "--stream-cache-mb=101", - // "--block-size-mb=102", - // "--buffer-size-mb=103", - // "--max-buffers=104", - // "--max-blocks-per-file=105", - // "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) - // - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has disabled streaming", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - // Enabled: pointers.Ptr(false), - // StreamCache: pointers.Ptr(uint64(101)), - // BlockSize: pointers.Ptr(uint64(102)), - // BufferSize: pointers.Ptr(uint64(103)), - // MaxBuffers: pointers.Ptr(uint64(104)), - // MaxBlocksPerFile: pointers.Ptr(uint64(105)), - // } - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, - // "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) - // pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) - // return deploymentVolumesTestScenario{ - // name: "Do not change existing PersistentVolume with class name, when creating new PVC", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // pvcForAnotherComponent, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherComponent, - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + type scenarioProperties struct { + changedNewRadixVolumeName string + changedNewRadixVolumeStorageName string + expectedVolumeName string + expectedNewSecretName string + expectedNewPvcName string + expectedNewPvName string + } + getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Update storage in existing volume name and storage", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.Name = scenarioProps.changedNewRadixVolumeName + vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) { + v.Name = scenarioProps.expectedVolumeName + }), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName + setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + changedNewRadixVolumeName: "volume101", + changedNewRadixVolumeStorageName: "storage101", + expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + expectedNewSecretName: "some-component-volume101-csiazurecreds", + expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + expectedNewPvName: "pv-radixvolumemount-some-uuid", + }), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) + pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) + pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + return deploymentVolumesTestScenario{ + name: "Garbage collect orphaned PVCs and PersistentVolume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createRandomPvc(props, props.namespace, props.componentName), + pvcForAnotherNamespace, + pvcForAnotherComponent, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherNamespace, + pvcForAnotherComponent, + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createRandomPv(props, props.namespace, props.componentName), + pvForAnotherNamespace, + pvForAnotherComponent, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + pvForAnotherNamespace, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Set readonly volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createRandomPvc(props, props.namespace, props.componentName), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createRandomPv(props, props.namespace, props.componentName), + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteOnce volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteMany volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadOnlyMany volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + pv.Spec.MountOptions = getMountOptions(props, true) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, + "--streaming=true", + "--stream-cache-mb=101", + "--block-size-mb=102", + "--buffer-size-mb=103", + "--max-buffers=104", + "--max-blocks-per-file=105", + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has disabled streaming", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + Enabled: pointers.Ptr(false), + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) + pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) + return deploymentVolumesTestScenario{ + name: "Do not change existing PersistentVolume with class name, when creating new PVC", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherComponent, + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) From 0ed485288e00b707a7c270cdf58392280c118c80 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 19 Dec 2024 14:48:46 +0100 Subject: [PATCH 34/68] Fixed comparison of existing auto-provisioned and manually created pv-s --- pkg/apis/deployment/volumemount.go | 4 +- pkg/apis/deployment/volumemount_test.go | 894 +++++++++--------- .../persistentvolume/persistentvolumeclaim.go | 36 +- 3 files changed, 498 insertions(+), 436 deletions(-) diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index e4cef3d57..46803a405 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -464,6 +464,7 @@ func buildPersistentVolumeClaim(appName, namespace, componentName, pvName string }, VolumeName: pvName, StorageClassName: pointers.Ptr(""), // use "" to avoid to use the "default" storage class + VolumeMode: pointers.Ptr(corev1.PersistentVolumeFilesystem), }, }, nil } @@ -566,7 +567,8 @@ func getCsiAzurePersistentVolumeAttributes(namespace string, radixVolumeMount *r attributes[persistentvolume.CsiVolumeMountAttributeSecretNamespace] = namespace } // Do not specify the key storage.kubernetes.io/csiProvisionerIdentity in csi.volumeAttributes in PV specification. This key indicates dynamically provisioned PVs - // It looks like: storage.kubernetes.io/csiProvisionerIdentity: 1731647415428-2825-blob.csi.azure.com + // https://github.com/kubernetes-csi/external-provisioner/blob/master/pkg/controller/controller.go#L289C5-L289C21 + // It looks like this: storage.kubernetes.io/csiProvisionerIdentity: 1731647415428-2825-blob.csi.azure.com return attributes } diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index cd7885e4f..c28eeda57 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -511,437 +511,474 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { environment = "some-env" componentName = "some-component" ) - anotherNamespace := commonUtils.RandString(10) + //anotherNamespace := commonUtils.RandString(10) anotherComponentName := commonUtils.RandString(10) var scenarios []deploymentVolumesTestScenario - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - type scenarioProperties struct { - changedNewRadixVolumeName string - changedNewRadixVolumeStorageName string - expectedVolumeName string - expectedNewSecretName string - expectedNewPvcName string - expectedNewPvName string - } - getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Update storage in existing volume name and storage", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.Name = scenarioProps.changedNewRadixVolumeName - vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) { - v.Name = scenarioProps.expectedVolumeName - }), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.ObjectMeta.Name = scenarioProps.expectedNewPvName - pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName - setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) - pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - changedNewRadixVolumeName: "volume101", - changedNewRadixVolumeStorageName: "storage101", - expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - expectedNewSecretName: "some-component-volume101-csiazurecreds", - expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - expectedNewPvName: "pv-radixvolumemount-some-uuid", - }), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) - pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) - pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) - matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - return deploymentVolumesTestScenario{ - name: "Garbage collect orphaned PVCs and PersistentVolume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - createRandomPv(props, props.namespace, props.componentName), - pvForAnotherNamespace, - pvForAnotherComponent, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - pvForAnotherNamespace, - pvForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Set readonly volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - createRandomPv(props, props.namespace, props.componentName), - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadWriteOnce volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - existingPvc, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - existingPv, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadWriteMany volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - existingPvc, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - existingPv, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadOnlyMany volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - }), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - pv.Spec.MountOptions = getMountOptions(props, true) - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, - "--streaming=true", - "--stream-cache-mb=101", - "--block-size-mb=102", - "--buffer-size-mb=103", - "--max-buffers=104", - "--max-blocks-per-file=105", - "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has disabled streaming", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - Enabled: pointers.Ptr(false), - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, - "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // type scenarioProperties struct { + // changedNewRadixVolumeName string + // changedNewRadixVolumeStorageName string + // expectedVolumeName string + // expectedNewSecretName string + // expectedNewPvcName string + // expectedNewPvName string + // } + // getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Update storage in existing volume name and storage", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.Name = scenarioProps.changedNewRadixVolumeName + // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) { + // v.Name = scenarioProps.expectedVolumeName + // }), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName + // setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + // pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + // changedNewRadixVolumeName: "volume101", + // changedNewRadixVolumeStorageName: "storage101", + // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + // expectedNewSecretName: "some-component-volume101-csiazurecreds", + // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + // expectedNewPvName: "pv-radixvolumemount-some-uuid", + // }), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) + // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + // pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) + // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + // matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // return deploymentVolumesTestScenario{ + // name: "Garbage collect orphaned PVCs and PersistentVolume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPv(props, props.namespace, props.componentName), + // pvForAnotherNamespace, + // pvForAnotherComponent, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // pvForAnotherNamespace, + // pvForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Set readonly volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPv(props, props.namespace, props.componentName), + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteOnce volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // existingPvc, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteMany volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // existingPvc, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadOnlyMany volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // }), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // pv.Spec.MountOptions = getMountOptions(props, true) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + // StreamCache: pointers.Ptr(uint64(101)), + // BlockSize: pointers.Ptr(uint64(102)), + // BufferSize: pointers.Ptr(uint64(103)), + // MaxBuffers: pointers.Ptr(uint64(104)), + // MaxBlocksPerFile: pointers.Ptr(uint64(105)), + // } + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, + // "--streaming=true", + // "--stream-cache-mb=101", + // "--block-size-mb=102", + // "--buffer-size-mb=103", + // "--max-buffers=104", + // "--max-blocks-per-file=105", + // "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) + // + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has disabled streaming", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + // Enabled: pointers.Ptr(false), + // StreamCache: pointers.Ptr(uint64(101)), + // BlockSize: pointers.Ptr(uint64(102)), + // BufferSize: pointers.Ptr(uint64(103)), + // MaxBuffers: pointers.Ptr(uint64(104)), + // MaxBlocksPerFile: pointers.Ptr(uint64(105)), + // } + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, + // "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) + // pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) + // return deploymentVolumesTestScenario{ + // name: "Do not change existing PersistentVolume with class name, when creating new PVC", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // pvcForAnotherComponent, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherComponent, + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + // return deploymentVolumesTestScenario{ + // name: "Do not change existing PersistentVolume without class name, when creating new PVC", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // pvcForAnotherComponent, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherComponent, + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) + existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) + expectedPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) return deploymentVolumesTestScenario{ - name: "Do not change existing PersistentVolume with class name, when creating new PVC", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - pvcForAnotherComponent, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - pvcForAnotherComponent, - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - existingPv, - pvForAnotherComponent, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - existingPv, - pvForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) - return deploymentVolumesTestScenario{ - name: "Do not change existing PersistentVolume without class name, when creating new PVC", + name: "Do not change existing PVC with class name, when creating new PersistentVolume", props: props, radixVolumeMounts: []v1.RadixVolumeMount{ createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), @@ -951,17 +988,17 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }, existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ pvcForAnotherComponent, + existingPvc, }, existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + existingPvc, pvcForAnotherComponent, }, existingPVsBeforeTestRun: []corev1.PersistentVolume{ - existingPv, pvForAnotherComponent, }, existingPVsAfterTestRun: []corev1.PersistentVolume{ - existingPv, + expectedPv, pvForAnotherComponent, }, } @@ -972,7 +1009,6 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }()...) suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { - //t.Parallel() for _, factory := range suite.radixCommonDeployComponentFactories[:1] { for _, scenario := range scenarios { t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) @@ -982,10 +1018,6 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { putExistingDeploymentVolumesScenarioDataToFakeCluster(&scenario, deployment) desiredDeployment := getDesiredDeployment(componentName, scenario.volumes) - //existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment) - //assert.Nil(t, err) - // - // action deployComponent := deployment.radixDeployment.Spec.Components[0] err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), &deployComponent, desiredDeployment) assert.Nil(t, err) @@ -1457,6 +1489,7 @@ func createExpectedPv(props expectedPvcPvProperties, modify func(pv *corev1.Pers StorageClassName: "", MountOptions: mountOptions, PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + VolumeMode: pointers.Ptr(corev1.PersistentVolumeFilesystem), }, Status: corev1.PersistentVolumeStatus{Phase: corev1.VolumeBound}, } @@ -1509,6 +1542,7 @@ func createAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties, modi StorageClassName: "some-storage-class", MountOptions: mountOptions, PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + VolumeMode: pointers.Ptr(corev1.PersistentVolumeFilesystem), }, Status: corev1.PersistentVolumeStatus{Phase: corev1.VolumeBound, LastPhaseTransitionTime: pointers.Ptr(metav1.Time{Time: time.Now()})}, } diff --git a/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go b/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go index 724643b8e..47111e04e 100644 --- a/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go +++ b/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go @@ -1,6 +1,7 @@ package persistentvolume import ( + "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/utils" corev1 "k8s.io/api/core/v1" ) @@ -24,18 +25,43 @@ func EqualPersistentVolumeClaims(pvc1, pvc2 *corev1.PersistentVolumeClaim) bool if pvc1.GetNamespace() != pvc2.GetNamespace() { return false } - if !utils.EqualStringMaps(pvc1.GetAnnotations(), pvc2.GetAnnotations()) { + if !utils.EqualStringMaps(getAnnotations(pvc1), getAnnotations(pvc2)) { return false } if !utils.EqualStringMaps(pvc1.GetLabels(), pvc2.GetLabels()) { return false } // ignore pvc1.Spec.StorageClassName != pvc2.Spec.StorageClassName for transition period - if pvc1.Spec.Resources.Requests[corev1.ResourceStorage] != pvc2.Spec.Resources.Requests[corev1.ResourceStorage] || - len(pvc1.Spec.AccessModes) != len(pvc2.Spec.AccessModes) || - (len(pvc1.Spec.AccessModes) == 1 && pvc1.Spec.AccessModes[0] != pvc2.Spec.AccessModes[0]) || - pvc1.Spec.VolumeMode != pvc2.Spec.VolumeMode { + pvc1StorageCapacity, existsPvc1StorageCapacity := pvc1.Spec.Resources.Requests[corev1.ResourceStorage] + pvc2StorageCapacity, existsPvc2StorageCapacity := pvc2.Spec.Resources.Requests[corev1.ResourceStorage] + if (existsPvc1StorageCapacity != existsPvc2StorageCapacity) || + (existsPvc1StorageCapacity && pvc1StorageCapacity.Cmp(pvc2StorageCapacity) != 0) { + return false + } + if len(pvc1.Spec.AccessModes) != len(pvc2.Spec.AccessModes) { + return false + } + if len(pvc1.Spec.AccessModes) == 1 && pvc1.Spec.AccessModes[0] != pvc2.Spec.AccessModes[0] { + return false + } + volumeMode1 := pointers.Val(pvc1.Spec.VolumeMode) + volumeMode2 := pointers.Val(pvc2.Spec.VolumeMode) + if volumeMode1 != volumeMode2 { return false } return true } + +func getAnnotations(pvc *corev1.PersistentVolumeClaim) map[string]string { + annotations := make(map[string]string) + for key, value := range pvc.GetAnnotations() { + if key == "pv.kubernetes.io/bind-completed" || + key == "pv.kubernetes.io/bound-by-controller" || + key == "volume.beta.kubernetes.io/storage-provisioner" || + key == "volume.kubernetes.io/storage-provisioner" { + continue // ignore automatically added annotation(s) + } + annotations[key] = value + } + return annotations +} From 5f881720e37bf62cc44526a79e576971557d1fad Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 19 Dec 2024 16:16:24 +0100 Subject: [PATCH 35/68] Removed deploy receiver from volumes --- pkg/apis/deployment/externaldns.go | 2 +- pkg/apis/deployment/kubedeployment.go | 2 +- pkg/apis/deployment/secrets.go | 54 +++-------- pkg/apis/deployment/volumemount.go | 121 ++++++++++++++---------- pkg/apis/deployment/volumemount_test.go | 2 +- 5 files changed, 88 insertions(+), 93 deletions(-) diff --git a/pkg/apis/deployment/externaldns.go b/pkg/apis/deployment/externaldns.go index 0ff3326b5..2bd45237c 100644 --- a/pkg/apis/deployment/externaldns.go +++ b/pkg/apis/deployment/externaldns.go @@ -82,7 +82,7 @@ func (deploy *Deployment) garbageCollectExternalDnsSecretsNoLongerInSpec(ctx con continue } - if err := deploy.deleteSecret(ctx, &secret); err != nil { + if err := deleteSecret(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment.Namespace, &secret); err != nil { return nil } } diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index 0a05025b7..819af8c81 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -37,7 +37,7 @@ func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployCompone return err } } - if err = deploy.createOrUpdateCsiAzureVolumeResources(ctx, deployComponent, desiredDeployment); err != nil { + if err = createOrUpdateCsiAzureVolumeResources(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace(), deployComponent, desiredDeployment); err != nil { return err } if err = deploy.handleJobAuxDeployment(ctx, deployComponent, desiredDeployment); err != nil { diff --git a/pkg/apis/deployment/secrets.go b/pkg/apis/deployment/secrets.go index 761df9d7e..a2d775733 100644 --- a/pkg/apis/deployment/secrets.go +++ b/pkg/apis/deployment/secrets.go @@ -15,6 +15,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" ) const ( @@ -67,13 +68,13 @@ func (deploy *Deployment) createOrUpdateSecretsForComponent(ctx context.Context, secretsToManage = append(secretsToManage, secretName) } - volumeMountSecretsToManage, err := deploy.createOrUpdateVolumeMountSecrets(ctx, namespace, component.GetName(), component.GetVolumeMounts()) + volumeMountSecretsToManage, err := CreateOrUpdateVolumeMountSecrets(ctx, deploy.kubeutil, deploy.registration.Name, namespace, component.GetName(), component.GetVolumeMounts()) if err != nil { return err } secretsToManage = append(secretsToManage, volumeMountSecretsToManage...) - err = deploy.garbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx, component, secretsToManage) + err = garbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx, deploy.kubeutil, namespace, component, secretsToManage) if err != nil { return err } @@ -113,19 +114,6 @@ func (deploy *Deployment) createOrUpdateSecretsForComponent(ctx context.Context, return nil } -func (deploy *Deployment) createOrUpdateVolumeMountSecrets(ctx context.Context, namespace, componentName string, volumeMounts []radixv1.RadixVolumeMount) ([]string, error) { - var volumeMountSecretsToManage []string - for _, volumeMount := range volumeMounts { - secretName, accountKey, accountName := deploy.getCsiAzureVolumeMountCredsSecrets(ctx, namespace, componentName, volumeMount.Name) - volumeMountSecretsToManage = append(volumeMountSecretsToManage, secretName) - err := deploy.createOrUpdateCsiAzureVolumeMountsSecrets(ctx, namespace, componentName, &volumeMount, secretName, accountName, accountKey) - if err != nil { - return nil, err - } - } - return volumeMountSecretsToManage, nil -} - func (deploy *Deployment) getBlobFuseCredsSecrets(ctx context.Context, ns, componentName, volumeMountName string) (string, []byte, []byte) { secretName := defaults.GetBlobFuseCredsSecretName(componentName, volumeMountName) accountKey := []byte(secretDefaultData) @@ -138,18 +126,6 @@ func (deploy *Deployment) getBlobFuseCredsSecrets(ctx context.Context, ns, compo return secretName, accountKey, accountName } -func (deploy *Deployment) getCsiAzureVolumeMountCredsSecrets(ctx context.Context, namespace, componentName, volumeMountName string) (string, []byte, []byte) { - secretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, volumeMountName) - accountKey := []byte(secretDefaultData) - accountName := []byte(secretDefaultData) - if deploy.kubeutil.SecretExists(ctx, namespace, secretName) { - oldSecret, _ := deploy.kubeutil.GetSecret(ctx, namespace, secretName) - accountKey = oldSecret.Data[defaults.CsiAzureCredsAccountKeyPart] - accountName = oldSecret.Data[defaults.CsiAzureCredsAccountNamePart] - } - return secretName, accountKey, accountName -} - func (deploy *Deployment) garbageCollectSecretsNoLongerInSpec(ctx context.Context) error { secrets, err := deploy.kubeutil.ListSecrets(ctx, deploy.radixDeployment.GetNamespace()) if err != nil { @@ -163,7 +139,7 @@ func (deploy *Deployment) garbageCollectSecretsNoLongerInSpec(ctx context.Contex } if deploy.isEligibleForGarbageCollectSecretsForComponent(existingSecret, componentName) { - err := deploy.deleteSecret(ctx, existingSecret) + err := deleteSecret(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment.GetNamespace(), existingSecret) if err != nil { return err } @@ -200,7 +176,7 @@ func (deploy *Deployment) garbageCollectSecretsNoLongerInSpecForComponent(ctx co } log.Ctx(ctx).Debug().Msgf("Delete secret %s no longer in spec for component %s", secret.Name, component.GetName()) - err = deploy.deleteSecret(ctx, secret) + err = deleteSecret(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment.GetNamespace(), secret) if err != nil { return err } @@ -210,20 +186,20 @@ func (deploy *Deployment) garbageCollectSecretsNoLongerInSpecForComponent(ctx co } func (deploy *Deployment) listSecretsForComponent(ctx context.Context, component radixv1.RadixCommonDeployComponent) ([]*v1.Secret, error) { - return deploy.listSecrets(ctx, getLabelSelectorForComponent(component)) + return listSecrets(ctx, deploy.kubeutil, deploy.radixDeployment.GetNamespace(), getLabelSelectorForComponent(component)) } -func (deploy *Deployment) listSecretsForVolumeMounts(ctx context.Context, component radixv1.RadixCommonDeployComponent) ([]*v1.Secret, error) { +func listSecretsForVolumeMounts(ctx context.Context, kubeUtil *kube.Kube, namespace string, component radixv1.RadixCommonDeployComponent) ([]*v1.Secret, error) { csiAzureVolumeMountSecret := getLabelSelectorForCsiAzureVolumeMountSecret(component) - csiSecrets, err := deploy.listSecrets(ctx, csiAzureVolumeMountSecret) + csiSecrets, err := listSecrets(ctx, kubeUtil, namespace, csiAzureVolumeMountSecret) if err != nil { return nil, err } return csiSecrets, nil } -func (deploy *Deployment) listSecrets(ctx context.Context, labelSelector string) ([]*v1.Secret, error) { - secrets, err := deploy.kubeutil.ListSecretsWithSelector(ctx, deploy.radixDeployment.GetNamespace(), labelSelector) +func listSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace, labelSelector string) ([]*v1.Secret, error) { + secrets, err := kubeUtil.ListSecretsWithSelector(ctx, namespace, labelSelector) if err != nil { return nil, err @@ -335,12 +311,12 @@ func (deploy *Deployment) removeOrphanedSecrets(ctx context.Context, ns, secretN } // GarbageCollectSecrets delete secrets, excluding with names in the excludeSecretNames -func (deploy *Deployment) GarbageCollectSecrets(ctx context.Context, secrets []*v1.Secret, excludeSecretNames []string) error { +func GarbageCollectSecrets(ctx context.Context, kubeClient kubernetes.Interface, namespace string, secrets []*v1.Secret, excludeSecretNames []string) error { for _, secret := range secrets { if slice.Any(excludeSecretNames, func(s string) bool { return s == secret.Name }) { continue } - err := deploy.deleteSecret(ctx, secret) + err := deleteSecret(ctx, kubeClient, namespace, secret) if err != nil { return err } @@ -348,12 +324,12 @@ func (deploy *Deployment) GarbageCollectSecrets(ctx context.Context, secrets []* return nil } -func (deploy *Deployment) deleteSecret(ctx context.Context, secret *v1.Secret) error { +func deleteSecret(ctx context.Context, kubeClient kubernetes.Interface, namespace string, secret *v1.Secret) error { log.Ctx(ctx).Debug().Msgf("Delete secret %s", secret.Name) - err := deploy.kubeclient.CoreV1().Secrets(deploy.radixDeployment.GetNamespace()).Delete(ctx, secret.Name, metav1.DeleteOptions{}) + err := kubeClient.CoreV1().Secrets(namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{}) if err != nil { return err } - log.Ctx(ctx).Info().Msgf("Deleted secret: %s in namespace %s", secret.GetName(), deploy.radixDeployment.GetNamespace()) + log.Ctx(ctx).Info().Msgf("Deleted secret: %s in namespace %s", secret.GetName(), namespace) return nil } diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/deployment/volumemount.go index 46803a405..f6a939729 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/deployment/volumemount.go @@ -364,13 +364,13 @@ func getCsiAzurePersistentVolumeName() string { return fmt.Sprintf(persistentvolume.CsiPersistentVolumeNameTemplate, uuid.New().String()) } -func (deploy *Deployment) createOrUpdateCsiAzureVolumeMountsSecrets(ctx context.Context, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, accountName, accountKey []byte) error { +func createOrUpdateCsiAzureVolumeMountsSecrets(ctx context.Context, kubeUtil *kube.Kube, appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, secretName string, accountName, accountKey []byte) error { secret := corev1.Secret{ Type: corev1.SecretTypeOpaque, ObjectMeta: metav1.ObjectMeta{ Name: secretName, Labels: map[string]string{ - kube.RadixAppLabel: deploy.registration.Name, + kube.RadixAppLabel: appName, kube.RadixComponentLabel: componentName, kube.RadixMountTypeLabel: string(GetCsiAzureVolumeMountType(radixVolumeMount)), kube.RadixVolumeMountNameLabel: radixVolumeMount.Name, @@ -385,7 +385,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeMountsSecrets(ctx context. secret.Data = data - _, err := deploy.kubeutil.ApplySecret(ctx, namespace, &secret) //nolint:staticcheck // must be updated to use UpdateSecret or CreateSecret + _, err := kubeUtil.ApplySecret(ctx, namespace, &secret) //nolint:staticcheck // must be updated to use UpdateSecret or CreateSecret if err != nil { return err } @@ -393,12 +393,12 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeMountsSecrets(ctx context. return nil } -func (deploy *Deployment) garbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx context.Context, component radixv1.RadixCommonDeployComponent, excludeSecretNames []string) error { - secrets, err := deploy.listSecretsForVolumeMounts(ctx, component) +func garbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx context.Context, kubeUtil *kube.Kube, namespace string, component radixv1.RadixCommonDeployComponent, excludeSecretNames []string) error { + secrets, err := listSecretsForVolumeMounts(ctx, kubeUtil, namespace, component) if err != nil { return err } - return deploy.GarbageCollectSecrets(ctx, secrets, excludeSecretNames) + return GarbageCollectSecrets(ctx, kubeUtil.KubeClient(), namespace, secrets, excludeSecretNames) } func getCsiPersistentVolumesForNamespace(ctx context.Context, kubeClient kubernetes.Interface, namespace string, onlyFunctional bool) ([]corev1.PersistentVolume, error) { @@ -423,16 +423,12 @@ func pvIsFunctional(pv corev1.PersistentVolume) bool { return slice.Any(functionalPersistentVolumePhases, func(phase corev1.PersistentVolumePhase) bool { return pv.Status.Phase == phase }) } -func (deploy *Deployment) getCsiAzurePersistentVolumeClaims(ctx context.Context, namespace, componentName string) (*corev1.PersistentVolumeClaimList, error) { - return deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ +func getCsiAzurePersistentVolumeClaims(ctx context.Context, kubeClient kubernetes.Interface, namespace, componentName string) (*corev1.PersistentVolumeClaimList, error) { + return kubeClient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ LabelSelector: getLabelSelectorForCsiAzurePersistenceVolumeClaim(componentName), }) } -func (deploy *Deployment) getPersistentVolumesForPvc(ctx context.Context) (*corev1.PersistentVolumeList, error) { - return deploy.kubeclient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) -} - func getLabelSelectorForCsiAzurePersistenceVolumeClaim(componentName string) string { return fmt.Sprintf("%s=%s, %s in (%s, %s)", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure)) } @@ -497,7 +493,7 @@ func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, persistentVolume.Spec.CSI = &corev1.CSIPersistentVolumeSource{ Driver: provisionerBlobCsiAzure, VolumeHandle: getVolumeHandle(namespace, componentName, pvName, getRadixVolumeMountStorage(radixVolumeMount)), - VolumeAttributes: getCsiAzurePersistentVolumeAttributes(namespace, radixVolumeMount, pvName, pvcName, useAzureIdentity, identityClientId, csiVolumeCredSecretName), + VolumeAttributes: getCsiAzurePersistentVolumeAttributes(namespace, radixVolumeMount, pvName, pvcName, useAzureIdentity, identityClientId), } if !useAzureIdentity { persistentVolume.Spec.CSI.NodeStageSecretRef = &corev1.SecretReference{Name: csiVolumeCredSecretName, Namespace: namespace} @@ -543,7 +539,7 @@ func getCsiAzurePersistentVolumeLabels(appName, namespace, componentName string, } } -func getCsiAzurePersistentVolumeAttributes(namespace string, radixVolumeMount *radixv1.RadixVolumeMount, pvName, pvcName string, useAzureIdentity bool, clientId, csiVolumeCredSecretName string) map[string]string { +func getCsiAzurePersistentVolumeAttributes(namespace string, radixVolumeMount *radixv1.RadixVolumeMount, pvName, pvcName string, useAzureIdentity bool, clientId string) map[string]string { attributes := make(map[string]string) switch GetCsiAzureVolumeMountType(radixVolumeMount) { case radixv1.MountTypeBlobFuse2FuseCsiAzure: @@ -686,23 +682,23 @@ func getRadixBlobFuse2VolumeMountBindingMode(radixVolumeMount *radixv1.RadixVolu return radixVolumeMount.BindingMode } -func (deploy *Deployment) deletePersistentVolumeClaim(ctx context.Context, namespace, pvcName string) error { +func deletePersistentVolumeClaim(ctx context.Context, kubeClient kubernetes.Interface, namespace, pvcName string) error { if len(namespace) == 0 || len(pvcName) == 0 { log.Ctx(ctx).Debug().Msgf("Skip deleting PVC - namespace %s or name %s is empty", namespace, pvcName) return nil } - if err := deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil { // && !k8serrors.IsNotFound(err) { + if err := kubeClient.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil { // && !k8serrors.IsNotFound(err) { return err } return nil } -func (deploy *Deployment) deletePersistentVolume(ctx context.Context, pvName string) error { +func deletePersistentVolume(ctx context.Context, kubeClient kubernetes.Interface, pvName string) error { if len(pvName) == 0 { log.Ctx(ctx).Debug().Msg("Skip deleting PersistentVolume - name is empty") return nil } - if err := deploy.kubeclient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { + if err := kubeClient.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { return err } return nil @@ -716,8 +712,8 @@ func getRadixVolumeMountStorage(radixVolumeMount *radixv1.RadixVolumeMount) stri return radixVolumeMount.Storage } -func (deploy *Deployment) garbageCollectOrphanedCsiAzurePersistentVolumes(ctx context.Context, excludePvcNames map[string]any) error { - pvList, err := deploy.kubeclient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) +func garbageCollectOrphanedCsiAzurePersistentVolumes(ctx context.Context, kubeClient kubernetes.Interface, excludePvcNames map[string]any) error { + pvList, err := kubeClient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) if err != nil { return err } @@ -733,7 +729,7 @@ func (deploy *Deployment) garbageCollectOrphanedCsiAzurePersistentVolumes(ctx co continue } log.Ctx(ctx).Info().Msgf("Delete orphaned Csi Azure PersistantVolume %s of PersistantVolumeClaim %s", pv.Name, pv.Spec.ClaimRef.Name) - if err := deploy.deletePersistentVolume(ctx, pv.Name); err != nil && !k8serrors.IsNotFound(err) { + if err := deletePersistentVolume(ctx, kubeClient, pv.Name); err != nil && !k8serrors.IsNotFound(err) { errs = append(errs, err) } } @@ -749,33 +745,31 @@ func knownCSIDriver(driver string) bool { } // createOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims, PersistentVolume -func (deploy *Deployment) createOrUpdateCsiAzureVolumeResources(ctx context.Context, deployComponent radixv1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) error { +func createOrUpdateCsiAzureVolumeResources(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, deployComponent radixv1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) error { componentName := deployComponent.GetName() - if err := deploy.createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx, componentName, deployComponent.GetIdentity(), desiredDeployment); err != nil { + if err := createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx, kubeClient, radixDeployment, namespace, componentName, deployComponent.GetIdentity(), desiredDeployment); err != nil { return err } - currentlyUsedPvcNames, err := deploy.getCurrentlyUsedPersistentVolumeClaims(ctx, desiredDeployment) + currentlyUsedPvcNames, err := getCurrentlyUsedPersistentVolumeClaims(ctx, kubeClient, radixDeployment, desiredDeployment) if err != nil { return err } - if err = deploy.garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx, componentName, currentlyUsedPvcNames); err != nil { + if err = garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx, kubeClient, namespace, componentName, currentlyUsedPvcNames); err != nil { return err } - err = deploy.garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, currentlyUsedPvcNames) - return err + return garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, kubeClient, currentlyUsedPvcNames) } -func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, componentName string, identity *radixv1.Identity, desiredDeployment *appsv1.Deployment) error { - namespace := deploy.radixDeployment.GetNamespace() - functionalPvList, err := getCsiPersistentVolumesForNamespace(ctx, deploy.kubeclient, namespace, true) +func createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace, componentName string, identity *radixv1.Identity, desiredDeployment *appsv1.Deployment) error { + functionalPvList, err := getCsiPersistentVolumesForNamespace(ctx, kubeClient, namespace, true) if err != nil { return err } - pvcByNameMap, err := deploy.getPvcByNameMap(ctx, namespace, componentName) + pvcByNameMap, err := getPvcByNameMap(ctx, kubeClient, namespace, componentName) if err != nil { return err } - radixVolumeMountsByNameMap := deploy.getRadixVolumeMountsByNameMap(componentName) + radixVolumeMountsByNameMap := getRadixVolumeMountsByNameMap(radixDeployment, componentName) var errs []error var volumes []corev1.Volume for _, volume := range desiredDeployment.Spec.Template.Spec.Volumes { @@ -787,7 +781,7 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx co if !existsRadixVolumeMount { continue } - processedVolume, err := deploy.createOrUpdateCsiAzureVolumeResourcesForVolume(ctx, namespace, componentName, identity, volume, radixVolumeMount, functionalPvList, pvcByNameMap) + processedVolume, err := createOrUpdateCsiAzureVolumeResourcesForVolume(ctx, kubeClient, radixDeployment, namespace, componentName, identity, volume, radixVolumeMount, functionalPvList, pvcByNameMap) if err != nil { errs = append(errs, err) continue @@ -803,11 +797,11 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx co return nil } -func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, namespace string, componentName string, identity *radixv1.Identity, volume corev1.Volume, radixVolumeMount *radixv1.RadixVolumeMount, functionalPvList []corev1.PersistentVolume, pvcByNameMap map[string]*corev1.PersistentVolumeClaim) (*corev1.Volume, error) { +func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, componentName string, identity *radixv1.Identity, volume corev1.Volume, radixVolumeMount *radixv1.RadixVolumeMount, functionalPvList []corev1.PersistentVolume, pvcByNameMap map[string]*corev1.PersistentVolumeClaim) (*corev1.Volume, error) { if volume.PersistentVolumeClaim == nil { return &volume, nil } - appName := deploy.radixDeployment.Spec.AppName + appName := radixDeployment.Spec.AppName pvcName := volume.PersistentVolumeClaim.ClaimName pvName, actualPvExists := getActualExistingCsiAzurePvNameByPvcName(appName, namespace, componentName, radixVolumeMount, functionalPvList, pvcName, identity) if !actualPvExists { @@ -824,14 +818,14 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolume(ctx con if !actualPvExists { log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) pv := populateCsiAzurePersistentVolume(&corev1.PersistentVolume{}, appName, namespace, componentName, pvName, pvcName, radixVolumeMount, identity) - if _, err = deploy.kubeclient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil { + if _, err = kubeClient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil { return nil, err } } if !pvcExist || !persistentvolume.EqualPersistentVolumeClaims(existingPvc, newPvc) { newPvc.SetName(pvcName) log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for PersistentVolume %s", newPvc.GetName(), namespace, pvName) - if _, err := deploy.kubeclient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, newPvc, metav1.CreateOptions{}); err != nil { + if _, err := kubeClient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, newPvc, metav1.CreateOptions{}); err != nil { return nil, err } } @@ -839,25 +833,25 @@ func (deploy *Deployment) createOrUpdateCsiAzureVolumeResourcesForVolume(ctx con return &volume, nil } -func (deploy *Deployment) getPvcByNameMap(ctx context.Context, namespace string, componentName string) (map[string]*corev1.PersistentVolumeClaim, error) { - pvcList, err := deploy.getCsiAzurePersistentVolumeClaims(ctx, namespace, componentName) +func getPvcByNameMap(ctx context.Context, kubeClient kubernetes.Interface, namespace string, componentName string) (map[string]*corev1.PersistentVolumeClaim, error) { + pvcList, err := getCsiAzurePersistentVolumeClaims(ctx, kubeClient, namespace, componentName) if err != nil { return nil, err } return persistentvolume.GetPersistentVolumeClaimMap(&pvcList.Items), nil } -func (deploy *Deployment) getCurrentlyUsedPersistentVolumeClaims(ctx context.Context, desiredDeployment *appsv1.Deployment) (map[string]any, error) { - namespace := deploy.radixDeployment.GetNamespace() +func getCurrentlyUsedPersistentVolumeClaims(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, desiredDeployment *appsv1.Deployment) (map[string]any, error) { + namespace := radixDeployment.GetNamespace() pvcNames := make(map[string]any) - deploymentList, err := deploy.kubeclient.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) + deploymentList, err := kubeClient.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) if err != nil { return nil, err } for _, deployment := range deploymentList.Items { pvcNames = appendUsedPersistenceVolumeClaimsFrom(pvcNames, deployment.Spec.Template.Spec.Volumes) } - jobsList, err := deploy.kubeclient.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{}) + jobsList, err := kubeClient.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{}) if err != nil { return nil, err } @@ -877,9 +871,8 @@ func appendUsedPersistenceVolumeClaimsFrom(pvcMap map[string]any, volumes []core }) } -func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx context.Context, componentName string, excludePvcNames map[string]any) error { - namespace := deploy.radixDeployment.GetNamespace() - pvcList, err := deploy.getCsiAzurePersistentVolumeClaims(ctx, namespace, componentName) +func garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx context.Context, kubeClient kubernetes.Interface, namespace, componentName string, excludePvcNames map[string]any) error { + pvcList, err := getCsiAzurePersistentVolumeClaims(ctx, kubeClient, namespace, componentName) if err != nil { return err } @@ -889,13 +882,13 @@ func (deploy *Deployment) garbageCollectCsiAzurePersistentVolumeClaimsAndPersist continue } log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolumeClaim %s in namespace %s", pvc.Name, namespace) - if err := deploy.deletePersistentVolumeClaim(ctx, namespace, pvc.Name); err != nil { + if err := deletePersistentVolumeClaim(ctx, kubeClient, namespace, pvc.Name); err != nil { errs = append(errs, err) continue } pvName := pvc.Spec.VolumeName log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolume %s in namespace %s", pvName, namespace) - if err := deploy.deletePersistentVolume(ctx, pvName); err != nil { + if err := deletePersistentVolume(ctx, kubeClient, pvName); err != nil { errs = append(errs, err) } } @@ -917,14 +910,14 @@ func getActualExistingCsiAzurePvNameByPvcName(appName, namespace, componentName return "", false } -func (deploy *Deployment) getRadixVolumeMountsByNameMap(componentName string) map[string]*radixv1.RadixVolumeMount { +func getRadixVolumeMountsByNameMap(radixDeployment *radixv1.RadixDeployment, componentName string) map[string]*radixv1.RadixVolumeMount { volumeMountsByNameMap := make(map[string]*radixv1.RadixVolumeMount) - for _, component := range deploy.radixDeployment.Spec.Components { + for _, component := range radixDeployment.Spec.Components { if findCsiAzureVolumeForComponent(volumeMountsByNameMap, component.VolumeMounts, componentName, &component) { break } } - for _, component := range deploy.radixDeployment.Spec.Jobs { + for _, component := range radixDeployment.Spec.Jobs { if findCsiAzureVolumeForComponent(volumeMountsByNameMap, component.VolumeMounts, componentName, &component) { break } @@ -966,3 +959,29 @@ func trimVolumeNameToValidLength(volumeName string) string { randString := strings.ToLower(commonUtils.RandStringStrSeed(randSize, volumeName)) return fmt.Sprintf("%s-%s", volumeName[:63-randSize-1], randString) } + +// CreateOrUpdateVolumeMountSecrets creates or updates secrets for volume mounts +func CreateOrUpdateVolumeMountSecrets(ctx context.Context, kubeUtil *kube.Kube, appName, namespace, componentName string, volumeMounts []radixv1.RadixVolumeMount) ([]string, error) { + var volumeMountSecretsToManage []string + for _, volumeMount := range volumeMounts { + secretName, accountKey, accountName := getCsiAzureVolumeMountCredsSecrets(ctx, kubeUtil, namespace, componentName, volumeMount.Name) + volumeMountSecretsToManage = append(volumeMountSecretsToManage, secretName) + err := createOrUpdateCsiAzureVolumeMountsSecrets(ctx, kubeUtil, appName, namespace, componentName, &volumeMount, secretName, accountName, accountKey) + if err != nil { + return nil, err + } + } + return volumeMountSecretsToManage, nil +} + +func getCsiAzureVolumeMountCredsSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace, componentName, volumeMountName string) (string, []byte, []byte) { + secretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, volumeMountName) + accountKey := []byte(secretDefaultData) + accountName := []byte(secretDefaultData) + if kubeUtil.SecretExists(ctx, namespace, secretName) { + oldSecret, _ := kubeUtil.GetSecret(ctx, namespace, secretName) + accountKey = oldSecret.Data[defaults.CsiAzureCredsAccountKeyPart] + accountName = oldSecret.Data[defaults.CsiAzureCredsAccountNamePart] + } + return secretName, accountKey, accountName +} diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/deployment/volumemount_test.go index c28eeda57..ab30df6bd 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/deployment/volumemount_test.go @@ -1019,7 +1019,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { desiredDeployment := getDesiredDeployment(componentName, scenario.volumes) deployComponent := deployment.radixDeployment.Spec.Components[0] - err := deployment.createOrUpdateCsiAzureVolumeResources(context.Background(), &deployComponent, desiredDeployment) + err := createOrUpdateCsiAzureVolumeResources(context.Background(), testEnv.kubeUtil.KubeClient(), deployment.radixDeployment, environment, &deployComponent, desiredDeployment) assert.Nil(t, err) existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment) From 36606ceecd3906c1ba72b5ec393d83a553e509f2 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 19 Dec 2024 16:59:26 +0100 Subject: [PATCH 36/68] Moving volumounts to own package --- pkg/apis/batch/kubejob.go | 5 +- pkg/apis/defaults/secrets.go | 6 + pkg/apis/deployment/deployment.go | 7 +- pkg/apis/deployment/environmentvariables.go | 3 +- pkg/apis/deployment/externaldns.go | 3 +- pkg/apis/deployment/kubedeployment.go | 12 +- pkg/apis/deployment/secretrefs.go | 4 +- pkg/apis/deployment/secrets.go | 68 +- pkg/apis/deployment/service.go | 3 +- pkg/apis/deployment/serviceaccountspec.go | 3 +- .../deployment/serviceaccountspec_test.go | 9 +- pkg/apis/deployment/volume_test.go | 37 + .../deployment/jobschedulercomponent.go | 38 +- .../volumemount.go | 57 +- .../volumemount_test.go | 1008 ++++++++--------- 15 files changed, 645 insertions(+), 618 deletions(-) create mode 100644 pkg/apis/deployment/volume_test.go rename pkg/apis/{ => internal}/deployment/jobschedulercomponent.go (61%) rename pkg/apis/{deployment => volumemount}/volumemount.go (95%) rename pkg/apis/{deployment => volumemount}/volumemount_test.go (67%) diff --git a/pkg/apis/batch/kubejob.go b/pkg/apis/batch/kubejob.go index 59cae9864..c48e23557 100644 --- a/pkg/apis/batch/kubejob.go +++ b/pkg/apis/batch/kubejob.go @@ -3,6 +3,7 @@ package batch import ( "context" "fmt" + "github.com/equinor/radix-operator/pkg/apis/volumemount" "strings" "github.com/equinor/radix-common/utils/numbers" @@ -180,7 +181,7 @@ func (s *syncer) getJobPodImagePullSecrets(rd *radixv1.RadixDeployment) []corev1 } func (s *syncer) getVolumes(ctx context.Context, namespace, environment string, batchJob *radixv1.RadixBatchJob, radixJobComponent *radixv1.RadixDeployJobComponent, radixDeploymentName string) ([]corev1.Volume, error) { - volumes, err := deployment.GetVolumes(ctx, s.kubeUtil, namespace, radixJobComponent, radixDeploymentName, nil) + volumes, err := volumemount.GetVolumes(ctx, s.kubeUtil, namespace, radixJobComponent, radixDeploymentName, nil) if err != nil { return nil, err } @@ -276,7 +277,7 @@ func getContainerPorts(radixJobComponent *radixv1.RadixDeployJobComponent) []cor } func (s *syncer) getContainerVolumeMounts(batchJob *radixv1.RadixBatchJob, radixJobComponent *radixv1.RadixDeployJobComponent, radixDeploymentName string) ([]corev1.VolumeMount, error) { - volumeMounts, err := deployment.GetRadixDeployComponentVolumeMounts(radixJobComponent, radixDeploymentName) + volumeMounts, err := volumemount.GetRadixDeployComponentVolumeMounts(radixJobComponent, radixDeploymentName) if err != nil { return nil, err } diff --git a/pkg/apis/defaults/secrets.go b/pkg/apis/defaults/secrets.go index 47255b7e2..2240fc6e6 100644 --- a/pkg/apis/defaults/secrets.go +++ b/pkg/apis/defaults/secrets.go @@ -70,6 +70,12 @@ const ( // TLSSecretName Secret name for the Radix wildcard TLS cert TLSSecretName = "radix-wildcard-tls-cert" + + // SecretDefaultData default secret content + SecretDefaultData = "xx" + + // SecretUsedBySecretStoreDriverLabel Label used to mark secrets used by secret store driver + SecretUsedBySecretStoreDriverLabel = "secrets-store.csi.k8s.io/used" ) // GetBlobFuseCredsSecretName Helper method diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index b9f617d19..e0e2f37fd 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "sort" "strings" "time" @@ -233,7 +234,7 @@ func (deploy *Deployment) syncDeployment(ctx context.Context) error { } for _, jobComponent := range deploy.radixDeployment.Spec.Jobs { ctx := log.Ctx(ctx).With().Str("jobComponent", jobComponent.Name).Logger().WithContext(ctx) - jobSchedulerComponent := newJobSchedulerComponent(&jobComponent, deploy.radixDeployment) + jobSchedulerComponent := internal.NewJobSchedulerComponent(&jobComponent, deploy.radixDeployment) if err := deploy.syncDeploymentForRadixComponent(ctx, jobSchedulerComponent); err != nil { errs = append(errs, err) } @@ -475,10 +476,6 @@ func getLabelSelectorForComponent(component v1.RadixCommonDeployComponent) strin return fmt.Sprintf("%s=%s", kube.RadixComponentLabel, component.GetName()) } -func getLabelSelectorForCsiAzureVolumeMountSecret(component v1.RadixCommonDeployComponent) string { - return fmt.Sprintf("%s=%s, %s in (%s, %s)", kube.RadixComponentLabel, component.GetName(), kube.RadixMountTypeLabel, string(v1.MountTypeBlobFuse2FuseCsiAzure), string(v1.MountTypeBlobFuse2Fuse2CsiAzure)) -} - func (deploy *Deployment) maintainHistoryLimit(ctx context.Context, deploymentHistoryLimit int) { if deploymentHistoryLimit <= 0 { return diff --git a/pkg/apis/deployment/environmentvariables.go b/pkg/apis/deployment/environmentvariables.go index 7a1fbc7d7..75b2120be 100644 --- a/pkg/apis/deployment/environmentvariables.go +++ b/pkg/apis/deployment/environmentvariables.go @@ -8,6 +8,7 @@ import ( radixmaps "github.com/equinor/radix-common/utils/maps" "github.com/equinor/radix-operator/pkg/apis/defaults" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "github.com/equinor/radix-operator/pkg/apis/kube" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" @@ -111,7 +112,7 @@ func getEnvironmentVariables(ctx context.Context, appName string, envVarsSource envVars = appendDefaultEnvVars(ctx, envVars, envVarsSource, currentEnvironment, namespace, appName, deployComponent) - if !isDeployComponentJobSchedulerDeployment(deployComponent) { // JobScheduler does not need env-vars for secrets and secret-refs + if !internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { // JobScheduler does not need env-vars for secrets and secret-refs envVars = append(envVars, utils.GetEnvVarsFromSecrets(deployComponent.GetName(), deployComponent.GetSecrets())...) envVars = append(envVars, utils.GetEnvVarsFromAzureKeyVaultSecretRefs(radixDeployment.GetName(), deployComponent.GetName(), deployComponent.GetSecretRefs())...) } diff --git a/pkg/apis/deployment/externaldns.go b/pkg/apis/deployment/externaldns.go index 2bd45237c..9aa2dc762 100644 --- a/pkg/apis/deployment/externaldns.go +++ b/pkg/apis/deployment/externaldns.go @@ -81,8 +81,7 @@ func (deploy *Deployment) garbageCollectExternalDnsSecretsNoLongerInSpec(ctx con if slice.Any(externalDnsAliases, func(rded radixv1.RadixDeployExternalDNS) bool { return rded.FQDN == fqdn }) { continue } - - if err := deleteSecret(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment.Namespace, &secret); err != nil { + if err := deploy.kubeutil.DeleteSecret(ctx, deploy.radixDeployment.Namespace, secret.Name); err != nil { return nil } } diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index 819af8c81..23f1b102a 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -2,6 +2,8 @@ package deployment import ( "context" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" + "github.com/equinor/radix-operator/pkg/apis/volumemount" commonUtils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/pointers" @@ -37,7 +39,7 @@ func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployCompone return err } } - if err = createOrUpdateCsiAzureVolumeResources(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace(), deployComponent, desiredDeployment); err != nil { + if err = volumemount.createOrUpdateCsiAzureVolumeResources(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace(), deployComponent, desiredDeployment); err != nil { return err } if err = deploy.handleJobAuxDeployment(ctx, deployComponent, desiredDeployment); err != nil { @@ -47,7 +49,7 @@ func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployCompone } func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) error { - if !isDeployComponentJobSchedulerDeployment(deployComponent) { + if !internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { return nil } currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, desiredDeployment) @@ -192,7 +194,7 @@ func (deploy *Deployment) getDeploymentPodLabels(deployComponent v1.RadixCommonD radixlabels.ForPodWithRadixIdentity(deployComponent.GetIdentity()), ) - if isDeployComponentJobSchedulerDeployment(deployComponent) { + if internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { labels = radixlabels.Merge(labels, radixlabels.ForPodIsJobScheduler()) } @@ -286,7 +288,7 @@ func (deploy *Deployment) setDesiredDeploymentProperties(ctx context.Context, de desiredDeployment.Spec.Template.Spec.Affinity = utils.GetAffinityForDeployComponent(ctx, deployComponent, appName, componentName) desiredDeployment.Spec.Template.Spec.Tolerations = utils.GetDeploymentPodSpecTolerations(deployComponent.GetNode()) - volumes, err := GetVolumes(ctx, deploy.kubeutil, deploy.getNamespace(), deployComponent, deploy.radixDeployment.GetName(), desiredDeployment.Spec.Template.Spec.Volumes) + volumes, err := volumemount.GetVolumes(ctx, deploy.kubeutil, deploy.getNamespace(), deployComponent, deploy.radixDeployment.GetName(), desiredDeployment.Spec.Template.Spec.Volumes) if err != nil { return err } @@ -302,7 +304,7 @@ func (deploy *Deployment) setDesiredDeploymentProperties(ctx context.Context, de return err } - volumeMounts, err := GetRadixDeployComponentVolumeMounts(deployComponent, deploy.radixDeployment.GetName()) + volumeMounts, err := volumemount.GetRadixDeployComponentVolumeMounts(deployComponent, deploy.radixDeployment.GetName()) if err != nil { return err } diff --git a/pkg/apis/deployment/secretrefs.go b/pkg/apis/deployment/secretrefs.go index 29b196a3c..8a0f16245 100644 --- a/pkg/apis/deployment/secretrefs.go +++ b/pkg/apis/deployment/secretrefs.go @@ -45,7 +45,7 @@ func (deploy *Deployment) getOrCreateSecretProviderClass(ctx context.Context, na secretProviderClass, err := deploy.kubeutil.GetSecretProviderClass(ctx, namespace, className) if err != nil { if errors.IsNotFound(err) { - return deploy.createAzureKeyVaultSecretProviderClassForRadixDeployment(ctx, namespace, appName, radixDeployComponentName, radixAzureKeyVault) + return deploy.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(ctx, namespace, appName, radixDeployComponentName, radixAzureKeyVault) } return nil, err } @@ -59,7 +59,7 @@ func (deploy *Deployment) getAzureKeyVaultCredsSecret(ctx context.Context, names return deploy.getOrCreateAzureKeyVaultCredsSecret(ctx, namespace, appName, radixDeployComponentName, azureKeyVault.Name) } -func (deploy *Deployment) createAzureKeyVaultSecretProviderClassForRadixDeployment(ctx context.Context, namespace string, appName string, radixDeployComponentName string, azureKeyVault radixv1.RadixAzureKeyVault) (*secretsstorev1.SecretProviderClass, error) { +func (deploy *Deployment) CreateAzureKeyVaultSecretProviderClassForRadixDeployment(ctx context.Context, namespace string, appName string, radixDeployComponentName string, azureKeyVault radixv1.RadixAzureKeyVault) (*secretsstorev1.SecretProviderClass, error) { radixDeploymentName := deploy.radixDeployment.GetName() tenantId := deploy.config.DeploymentSyncer.TenantID identity := deploy.getIdentityFromRadixCommonDeployComponent(radixDeployComponentName) diff --git a/pkg/apis/deployment/secrets.go b/pkg/apis/deployment/secrets.go index a2d775733..6062fb0f7 100644 --- a/pkg/apis/deployment/secrets.go +++ b/pkg/apis/deployment/secrets.go @@ -3,6 +3,7 @@ package deployment import ( "context" "fmt" + "github.com/equinor/radix-operator/pkg/apis/volumemount" "strings" "github.com/equinor/radix-common/utils/slice" @@ -15,12 +16,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -const ( - secretDefaultData = "xx" - secretUsedBySecretStoreDriverLabel = "secrets-store.csi.k8s.io/used" ) func tlsSecretDefaultData() map[string][]byte { @@ -68,13 +63,13 @@ func (deploy *Deployment) createOrUpdateSecretsForComponent(ctx context.Context, secretsToManage = append(secretsToManage, secretName) } - volumeMountSecretsToManage, err := CreateOrUpdateVolumeMountSecrets(ctx, deploy.kubeutil, deploy.registration.Name, namespace, component.GetName(), component.GetVolumeMounts()) + volumeMountSecretsToManage, err := volumemount.CreateOrUpdateVolumeMountSecrets(ctx, deploy.kubeutil, deploy.registration.Name, namespace, component.GetName(), component.GetVolumeMounts()) if err != nil { return err } secretsToManage = append(secretsToManage, volumeMountSecretsToManage...) - err = garbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx, deploy.kubeutil, namespace, component, secretsToManage) + err = volumemount.garbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx, deploy.kubeutil, namespace, component, secretsToManage) if err != nil { return err } @@ -116,8 +111,8 @@ func (deploy *Deployment) createOrUpdateSecretsForComponent(ctx context.Context, func (deploy *Deployment) getBlobFuseCredsSecrets(ctx context.Context, ns, componentName, volumeMountName string) (string, []byte, []byte) { secretName := defaults.GetBlobFuseCredsSecretName(componentName, volumeMountName) - accountKey := []byte(secretDefaultData) - accountName := []byte(secretDefaultData) + accountKey := []byte(defaults.SecretDefaultData) + accountName := []byte(defaults.SecretDefaultData) if deploy.kubeutil.SecretExists(ctx, ns, secretName) { oldSecret, _ := deploy.kubeutil.GetSecret(ctx, ns, secretName) accountKey = oldSecret.Data[defaults.BlobFuseCredsAccountKeyPart] @@ -139,8 +134,7 @@ func (deploy *Deployment) garbageCollectSecretsNoLongerInSpec(ctx context.Contex } if deploy.isEligibleForGarbageCollectSecretsForComponent(existingSecret, componentName) { - err := deleteSecret(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment.GetNamespace(), existingSecret) - if err != nil { + if err := deploy.kubeutil.DeleteSecret(ctx, deploy.radixDeployment.GetNamespace(), existingSecret.Name); err != nil && !errors.IsNotFound(err) { return err } } @@ -176,8 +170,7 @@ func (deploy *Deployment) garbageCollectSecretsNoLongerInSpecForComponent(ctx co } log.Ctx(ctx).Debug().Msgf("Delete secret %s no longer in spec for component %s", secret.Name, component.GetName()) - err = deleteSecret(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment.GetNamespace(), secret) - if err != nil { + if err = deploy.kubeutil.DeleteSecret(ctx, deploy.radixDeployment.GetNamespace(), secret.Name); err != nil && !errors.IsNotFound(err) { return err } } @@ -189,15 +182,6 @@ func (deploy *Deployment) listSecretsForComponent(ctx context.Context, component return listSecrets(ctx, deploy.kubeutil, deploy.radixDeployment.GetNamespace(), getLabelSelectorForComponent(component)) } -func listSecretsForVolumeMounts(ctx context.Context, kubeUtil *kube.Kube, namespace string, component radixv1.RadixCommonDeployComponent) ([]*v1.Secret, error) { - csiAzureVolumeMountSecret := getLabelSelectorForCsiAzureVolumeMountSecret(component) - csiSecrets, err := listSecrets(ctx, kubeUtil, namespace, csiAzureVolumeMountSecret) - if err != nil { - return nil, err - } - return csiSecrets, nil -} - func listSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace, labelSelector string) ([]*v1.Secret, error) { secrets, err := kubeUtil.ListSecretsWithSelector(ctx, namespace, labelSelector) @@ -244,17 +228,17 @@ func buildAzureKeyVaultCredentialsSecret(appName, componentName, secretName, azK ObjectMeta: metav1.ObjectMeta{ Name: secretName, Labels: map[string]string{ - kube.RadixAppLabel: appName, - kube.RadixComponentLabel: componentName, - kube.RadixSecretRefTypeLabel: string(radixv1.RadixSecretRefTypeAzureKeyVault), - kube.RadixSecretRefNameLabel: strings.ToLower(azKeyVaultName), - secretUsedBySecretStoreDriverLabel: "true", // used by CSI Azure Key vault secret store driver for secret rotation + kube.RadixAppLabel: appName, + kube.RadixComponentLabel: componentName, + kube.RadixSecretRefTypeLabel: string(radixv1.RadixSecretRefTypeAzureKeyVault), + kube.RadixSecretRefNameLabel: strings.ToLower(azKeyVaultName), + defaults.SecretUsedBySecretStoreDriverLabel: "true", // used by CSI Azure Key vault secret store driver for secret rotation }, }, } data := make(map[string][]byte) - defaultValue := []byte(secretDefaultData) + defaultValue := []byte(defaults.SecretDefaultData) data["clientid"] = defaultValue data["clientsecret"] = defaultValue @@ -274,7 +258,7 @@ func (deploy *Deployment) createClientCertificateSecret(ctx context.Context, ns, }, } - defaultValue := []byte(secretDefaultData) + defaultValue := []byte(defaults.SecretDefaultData) // Will need to set fake data in order to apply the secret. The user then need to set data to real values data := make(map[string][]byte) @@ -309,27 +293,3 @@ func (deploy *Deployment) removeOrphanedSecrets(ctx context.Context, ns, secretN return nil } - -// GarbageCollectSecrets delete secrets, excluding with names in the excludeSecretNames -func GarbageCollectSecrets(ctx context.Context, kubeClient kubernetes.Interface, namespace string, secrets []*v1.Secret, excludeSecretNames []string) error { - for _, secret := range secrets { - if slice.Any(excludeSecretNames, func(s string) bool { return s == secret.Name }) { - continue - } - err := deleteSecret(ctx, kubeClient, namespace, secret) - if err != nil { - return err - } - } - return nil -} - -func deleteSecret(ctx context.Context, kubeClient kubernetes.Interface, namespace string, secret *v1.Secret) error { - log.Ctx(ctx).Debug().Msgf("Delete secret %s", secret.Name) - err := kubeClient.CoreV1().Secrets(namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - log.Ctx(ctx).Info().Msgf("Deleted secret: %s in namespace %s", secret.GetName(), namespace) - return nil -} diff --git a/pkg/apis/deployment/service.go b/pkg/apis/deployment/service.go index ed8c737b9..562b35bd0 100644 --- a/pkg/apis/deployment/service.go +++ b/pkg/apis/deployment/service.go @@ -2,6 +2,7 @@ package deployment import ( "context" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "github.com/equinor/radix-operator/pkg/apis/kube" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" @@ -59,7 +60,7 @@ func getServiceConfig(component v1.RadixCommonDeployComponent, radixDeployment * } selector := map[string]string{kube.RadixComponentLabel: component.GetName()} - if isDeployComponentJobSchedulerDeployment(component) { + if internal.IsDeployComponentJobSchedulerDeployment(component) { selector[kube.RadixPodIsJobSchedulerLabel] = "true" } diff --git a/pkg/apis/deployment/serviceaccountspec.go b/pkg/apis/deployment/serviceaccountspec.go index 508c254ac..b63deee63 100644 --- a/pkg/apis/deployment/serviceaccountspec.go +++ b/pkg/apis/deployment/serviceaccountspec.go @@ -3,6 +3,7 @@ package deployment import ( "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/defaults" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" ) @@ -67,7 +68,7 @@ func (spec *radixComponentServiceAccountSpec) AutomountServiceAccountToken() *bo // NewServiceAccountSpec Create ServiceAccountSpec based on RadixDeployment and RadixCommonDeployComponent func NewServiceAccountSpec(radixDeploy *v1.RadixDeployment, deployComponent v1.RadixCommonDeployComponent) ServiceAccountSpec { isComponent := deployComponent.GetType() == v1.RadixComponentTypeComponent - isJobScheduler := isDeployComponentJobSchedulerDeployment(deployComponent) + isJobScheduler := internal.IsDeployComponentJobSchedulerDeployment(deployComponent) if isComponent && isRadixAPI(radixDeploy) { return &radixAPIServiceAccountSpec{} diff --git a/pkg/apis/deployment/serviceaccountspec_test.go b/pkg/apis/deployment/serviceaccountspec_test.go index 66520f7a4..1c32ba3c0 100644 --- a/pkg/apis/deployment/serviceaccountspec_test.go +++ b/pkg/apis/deployment/serviceaccountspec_test.go @@ -1,6 +1,7 @@ package deployment import ( + deployment2 "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "testing" "github.com/equinor/radix-common/utils/pointers" @@ -40,7 +41,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(false), spec.AutomountServiceAccountToken()) assert.Equal(t, utils.GetComponentServiceAccountName(rd.Spec.Components[1].Name), spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) + spec = NewServiceAccountSpec(rd, deployment2.newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) @@ -48,7 +49,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(false), spec.AutomountServiceAccountToken()) assert.Equal(t, defaultServiceAccountName, spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, newJobSchedulerComponent(&rd.Spec.Jobs[1], rd)) + spec = NewServiceAccountSpec(rd, deployment2.newJobSchedulerComponent(&rd.Spec.Jobs[1], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) @@ -71,7 +72,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixAPIServiceAccountName, spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) + spec = NewServiceAccountSpec(rd, deployment2.newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) @@ -95,7 +96,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixGithubWebhookServiceAccountName, spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) + spec = NewServiceAccountSpec(rd, deployment2.newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) diff --git a/pkg/apis/deployment/volume_test.go b/pkg/apis/deployment/volume_test.go new file mode 100644 index 000000000..d3871abf6 --- /dev/null +++ b/pkg/apis/deployment/volume_test.go @@ -0,0 +1,37 @@ +package deployment + +import ( + "context" + "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/resource" + v2 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func Test_EmptyDir(t *testing.T) { + appName, envName, compName := "anyapp", "anyenv", "anycomp" + + tu, kubeclient, kubeUtil, radixclient, kedaClient, prometheusclient, _, certClient := SetupTest(t) + builder := utils.NewDeploymentBuilder(). + WithRadixApplication(utils.NewRadixApplicationBuilder().WithAppName(appName).WithRadixRegistration(utils.NewRegistrationBuilder().WithName(appName))). + WithAppName(appName). + WithEnvironment(envName). + WithComponents( + utils.NewDeployComponentBuilder().WithName(compName).WithVolumeMounts( + v1.RadixVolumeMount{Name: "cache", Path: "/cache", EmptyDir: &v1.RadixEmptyDirVolumeMount{SizeLimit: resource.MustParse("50M")}}, + v1.RadixVolumeMount{Name: "log", Path: "/log", EmptyDir: &v1.RadixEmptyDirVolumeMount{SizeLimit: resource.MustParse("100M")}}, + ), + ) + + rd, err := ApplyDeploymentWithSync(tu, kubeclient, kubeUtil, radixclient, kedaClient, prometheusclient, certClient, builder) + require.NoError(t, err) + assert.NotNil(t, rd) + + deployment, err := kubeclient.AppsV1().Deployments(utils.GetEnvironmentNamespace(appName, envName)).Get(context.Background(), compName, v2.GetOptions{}) + require.NoError(t, err) + assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 2) + assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2) +} diff --git a/pkg/apis/deployment/jobschedulercomponent.go b/pkg/apis/internal/deployment/jobschedulercomponent.go similarity index 61% rename from pkg/apis/deployment/jobschedulercomponent.go rename to pkg/apis/internal/deployment/jobschedulercomponent.go index 514185110..557856c5a 100644 --- a/pkg/apis/deployment/jobschedulercomponent.go +++ b/pkg/apis/internal/deployment/jobschedulercomponent.go @@ -9,30 +9,31 @@ import ( radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" ) -type jobSchedulerComponent struct { +type JobSchedulerComponent struct { *radixv1.RadixDeployJobComponent - radixDeployment *radixv1.RadixDeployment + RadixDeployment *radixv1.RadixDeployment } -func newJobSchedulerComponent(jobComponent *radixv1.RadixDeployJobComponent, rd *radixv1.RadixDeployment) radixv1.RadixCommonDeployComponent { - return &jobSchedulerComponent{ +// NewJobSchedulerComponent Constructor +func NewJobSchedulerComponent(jobComponent *radixv1.RadixDeployJobComponent, rd *radixv1.RadixDeployment) radixv1.RadixCommonDeployComponent { + return &JobSchedulerComponent{ jobComponent, rd, } } -func (js *jobSchedulerComponent) GetHealthChecks() *radixv1.RadixHealthChecks { +func (js *JobSchedulerComponent) GetHealthChecks() *radixv1.RadixHealthChecks { return nil } -func (js *jobSchedulerComponent) GetImage() string { +func (js *JobSchedulerComponent) GetImage() string { containerRegistry := os.Getenv(defaults.ContainerRegistryEnvironmentVariable) radixJobScheduler := os.Getenv(defaults.OperatorRadixJobSchedulerEnvironmentVariable) radixJobSchedulerImageUrl := fmt.Sprintf("%s/%s", containerRegistry, radixJobScheduler) return radixJobSchedulerImageUrl } -func (js *jobSchedulerComponent) GetPorts() []radixv1.ComponentPort { +func (js *JobSchedulerComponent) GetPorts() []radixv1.ComponentPort { if js.RadixDeployJobComponent.SchedulerPort == nil { return nil } @@ -45,25 +46,25 @@ func (js *jobSchedulerComponent) GetPorts() []radixv1.ComponentPort { } } -func (js *jobSchedulerComponent) GetEnvironmentVariables() radixv1.EnvVarsMap { +func (js *JobSchedulerComponent) GetEnvironmentVariables() radixv1.EnvVarsMap { envVarsMap := js.EnvironmentVariables.DeepCopy() if envVarsMap == nil { envVarsMap = radixv1.EnvVarsMap{} } - envVarsMap[defaults.RadixDeploymentEnvironmentVariable] = js.radixDeployment.Name + envVarsMap[defaults.RadixDeploymentEnvironmentVariable] = js.RadixDeployment.Name envVarsMap[defaults.OperatorEnvLimitDefaultMemoryEnvironmentVariable] = os.Getenv(defaults.OperatorEnvLimitDefaultMemoryEnvironmentVariable) return envVarsMap } -func (js *jobSchedulerComponent) GetSecrets() []string { +func (js *JobSchedulerComponent) GetSecrets() []string { return nil } -func (js *jobSchedulerComponent) GetMonitoring() bool { +func (js *JobSchedulerComponent) GetMonitoring() bool { return false } -func (js *jobSchedulerComponent) GetResources() *radixv1.ResourceRequirements { +func (js *JobSchedulerComponent) GetResources() *radixv1.ResourceRequirements { return &radixv1.ResourceRequirements{ Limits: map[string]string{ "memory": "500M", @@ -75,25 +76,26 @@ func (js *jobSchedulerComponent) GetResources() *radixv1.ResourceRequirements { } } -func (js *jobSchedulerComponent) GetReadOnlyFileSystem() *bool { +func (js *JobSchedulerComponent) GetReadOnlyFileSystem() *bool { return pointers.Ptr(true) } -func (js *jobSchedulerComponent) IsAlwaysPullImageOnDeploy() bool { +func (js *JobSchedulerComponent) IsAlwaysPullImageOnDeploy() bool { return true } -func (js *jobSchedulerComponent) GetNode() *radixv1.RadixNode { +func (js *JobSchedulerComponent) GetNode() *radixv1.RadixNode { // Job configuration in radixconfig.yaml contains section "node", which supposed to configure scheduled jobs by RadixDeployment // "node" section settings should not be applied to the JobScheduler component itself return nil } -func (js *jobSchedulerComponent) GetRuntime() *radixv1.Runtime { +func (js *JobSchedulerComponent) GetRuntime() *radixv1.Runtime { return &radixv1.Runtime{Architecture: radixv1.RuntimeArchitectureAmd64} } -func isDeployComponentJobSchedulerDeployment(deployComponent radixv1.RadixCommonDeployComponent) bool { - _, isJobScheduler := interface{}(deployComponent).(*jobSchedulerComponent) +// IsDeployComponentJobSchedulerDeployment Checks if deployComponent is a JobScheduler deployment +func IsDeployComponentJobSchedulerDeployment(deployComponent radixv1.RadixCommonDeployComponent) bool { + _, isJobScheduler := interface{}(deployComponent).(*JobSchedulerComponent) return isJobScheduler } diff --git a/pkg/apis/deployment/volumemount.go b/pkg/apis/volumemount/volumemount.go similarity index 95% rename from pkg/apis/deployment/volumemount.go rename to pkg/apis/volumemount/volumemount.go index f6a939729..b72a6750e 100644 --- a/pkg/apis/deployment/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -1,4 +1,4 @@ -package deployment +package volumemount import ( "context" @@ -12,6 +12,7 @@ import ( "github.com/equinor/radix-common/utils/slice" "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/defaults/k8s" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "github.com/equinor/radix-operator/pkg/apis/internal/persistentvolume" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" @@ -70,7 +71,7 @@ func GetRadixDeployComponentVolumeMounts(deployComponent radixv1.RadixCommonDepl } func getRadixComponentVolumeMounts(deployComponent radixv1.RadixCommonDeployComponent) ([]corev1.VolumeMount, error) { - if isDeployComponentJobSchedulerDeployment(deployComponent) { + if internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { return nil, nil } @@ -398,7 +399,16 @@ func garbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx context.Con if err != nil { return err } - return GarbageCollectSecrets(ctx, kubeUtil.KubeClient(), namespace, secrets, excludeSecretNames) + for _, secret := range secrets { + if slice.Any(excludeSecretNames, func(s string) bool { return s == secret.Name }) { + continue + } + if err := kubeUtil.DeleteSecret(ctx, namespace, secret.Name); err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + + return garbageCollectSecrets(ctx, kubeUtil.KubeClient(), namespace, secrets, excludeSecretNames) } func getCsiPersistentVolumesForNamespace(ctx context.Context, kubeClient kubernetes.Interface, namespace string, onlyFunctional bool) ([]corev1.PersistentVolume, error) { @@ -976,8 +986,8 @@ func CreateOrUpdateVolumeMountSecrets(ctx context.Context, kubeUtil *kube.Kube, func getCsiAzureVolumeMountCredsSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace, componentName, volumeMountName string) (string, []byte, []byte) { secretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, volumeMountName) - accountKey := []byte(secretDefaultData) - accountName := []byte(secretDefaultData) + accountKey := []byte(defaults.SecretDefaultData) + accountName := []byte(defaults.SecretDefaultData) if kubeUtil.SecretExists(ctx, namespace, secretName) { oldSecret, _ := kubeUtil.GetSecret(ctx, namespace, secretName) accountKey = oldSecret.Data[defaults.CsiAzureCredsAccountKeyPart] @@ -985,3 +995,40 @@ func getCsiAzureVolumeMountCredsSecrets(ctx context.Context, kubeUtil *kube.Kube } return secretName, accountKey, accountName } + +func getLabelSelectorForCsiAzureVolumeMountSecret(component radixv1.RadixCommonDeployComponent) string { + return fmt.Sprintf("%s=%s, %s in (%s, %s)", kube.RadixComponentLabel, component.GetName(), kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure)) +} + +func listSecretsForVolumeMounts(ctx context.Context, kubeUtil *kube.Kube, namespace string, component radixv1.RadixCommonDeployComponent) ([]*corev1.Secret, error) { + csiAzureVolumeMountSecret := getLabelSelectorForCsiAzureVolumeMountSecret(component) + csiSecrets, err := kubeUtil.ListSecretsWithSelector(ctx, namespace, csiAzureVolumeMountSecret) + if err != nil { + return nil, err + } + return csiSecrets, nil +} + +// garbageCollectSecrets delete secrets, excluding with names in the excludeSecretNames +func garbageCollectSecrets(ctx context.Context, kubeClient kubernetes.Interface, namespace string, secrets []*corev1.Secret, excludeSecretNames []string) error { + for _, secret := range secrets { + if slice.Any(excludeSecretNames, func(s string) bool { return s == secret.Name }) { + continue + } + err := DeleteSecret(ctx, kubeClient, namespace, secret) + if err != nil { + return err + } + } + return nil +} + +func DeleteSecret(ctx context.Context, kubeClient kubernetes.Interface, namespace string, secret *corev1.Secret) error { + log.Ctx(ctx).Debug().Msgf("Delete secret %s", secret.Name) + err := kubeClient.CoreV1().Secrets(namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{}) + if err != nil { + return err + } + log.Ctx(ctx).Info().Msgf("Deleted secret: %s in namespace %s", secret.GetName(), namespace) + return nil +} diff --git a/pkg/apis/deployment/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go similarity index 67% rename from pkg/apis/deployment/volumemount_test.go rename to pkg/apis/volumemount/volumemount_test.go index ab30df6bd..f7a05f535 100644 --- a/pkg/apis/deployment/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -1,9 +1,10 @@ -package deployment +package volumemount import ( "context" "encoding/json" "fmt" + "github.com/equinor/radix-operator/pkg/apis/deployment" "github.com/google/uuid" "k8s.io/apimachinery/pkg/types" "strings" @@ -101,8 +102,8 @@ func getTestEnv() TestEnv { return testEnv } -func getDeployment(testEnv TestEnv) *Deployment { - return &Deployment{ +func getDeployment(testEnv TestEnv) *deployment.Deployment { + return &deployment.Deployment{ kubeclient: testEnv.kubeclient, radixclient: testEnv.radixclient, kubeutil: testEnv.kubeUtil, @@ -376,10 +377,10 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { for _, factory := range suite.radixCommonDeployComponentFactories { t.Logf("Test case for component %s", factory.GetTargetType()) - deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{}) - deployComponent := deployment.radixDeployment.Spec.Components[0] + radixDeployment := buildRd(appName, environment, componentName, []v1.RadixVolumeMount{}) + deployComponent := radixDeployment.Spec.Components[0] - volumes, err := GetVolumes(context.Background(), deployment.kubeutil, deployment.getNamespace(), &deployComponent, deployment.radixDeployment.GetName(), nil) + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, radixDeployment.GetNamespace(), &deployComponent, radixDeployment.GetName(), nil) assert.Nil(t, err) assert.Len(t, volumes, 0) @@ -393,10 +394,10 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { for _, scenario := range scenarios { t.Logf("Test case %s for component %s", scenario.name, factory.GetTargetType()) - deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{scenario.radixVolumeMount}) - deployComponent := deployment.radixDeployment.Spec.Components[0] + radixDeployment := buildRd(appName, environment, componentName, []v1.RadixVolumeMount{scenario.radixVolumeMount}) + deployComponent := radixDeployment.Spec.Components[0] - volumes, err := GetVolumes(context.Background(), deployment.kubeutil, deployment.getNamespace(), &deployComponent, deployment.radixDeployment.GetName(), nil) + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, radixDeployment.GetNamespace(), &deployComponent, radixDeployment.GetName(), nil) assert.Nil(t, err) assert.Len(t, volumes, 1) @@ -455,13 +456,11 @@ func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { suite.T().Run("No volumes", func(t *testing.T) { t.Parallel() - testEnv := getTestEnv() - deployment := getDeployment(testEnv) for _, factory := range suite.radixCommonDeployComponentFactories { t.Logf("Test case for component %s", factory.GetTargetType()) - deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{}) - deployComponent := deployment.radixDeployment.Spec.Components[0] + radixDeployment := buildRd(appName, environment, componentName, []v1.RadixVolumeMount{}) + deployComponent := radixDeployment.Spec.Components[0] volumes, err := GetRadixDeployComponentVolumeMounts(&deployComponent, "") @@ -471,14 +470,12 @@ func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { }) suite.T().Run("Exists volume", func(t *testing.T) { t.Parallel() - testEnv := getTestEnv() - deployment := getDeployment(testEnv) for _, factory := range suite.radixCommonDeployComponentFactories { for _, scenario := range scenarios { t.Logf("Test case %s for component %s", scenario.name, factory.GetTargetType()) - deployment.radixDeployment = buildRd(appName, environment, componentName, []v1.RadixVolumeMount{scenario.radixVolumeMount}) - deployComponent := deployment.radixDeployment.Spec.Components[0] + radixDeployment := buildRd(appName, environment, componentName, []v1.RadixVolumeMount{scenario.radixVolumeMount}) + deployComponent := radixDeployment.Spec.Components[0] volumeMounts, err := GetRadixDeployComponentVolumeMounts(&deployComponent, "") @@ -511,465 +508,465 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { environment = "some-env" componentName = "some-component" ) - //anotherNamespace := commonUtils.RandString(10) + anotherNamespace := commonUtils.RandString(10) anotherComponentName := commonUtils.RandString(10) var scenarios []deploymentVolumesTestScenario - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // type scenarioProperties struct { - // changedNewRadixVolumeName string - // changedNewRadixVolumeStorageName string - // expectedVolumeName string - // expectedNewSecretName string - // expectedNewPvcName string - // expectedNewPvName string - // } - // getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Update storage in existing volume name and storage", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.Name = scenarioProps.changedNewRadixVolumeName - // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) { - // v.Name = scenarioProps.expectedVolumeName - // }), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName - // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName - // setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) - // pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - // changedNewRadixVolumeName: "volume101", - // changedNewRadixVolumeStorageName: "storage101", - // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - // expectedNewSecretName: "some-component-volume101-csiazurecreds", - // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - // expectedNewPvName: "pv-radixvolumemount-some-uuid", - // }), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) - // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - // pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) - // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - // matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // return deploymentVolumesTestScenario{ - // name: "Garbage collect orphaned PVCs and PersistentVolume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPv(props, props.namespace, props.componentName), - // pvForAnotherNamespace, - // pvForAnotherComponent, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // pvForAnotherNamespace, - // pvForAnotherComponent, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Set readonly volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPv(props, props.namespace, props.componentName), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteOnce volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // existingPvc, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteMany volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // existingPvc, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadOnlyMany volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // }), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // pv.Spec.MountOptions = getMountOptions(props, true) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - // StreamCache: pointers.Ptr(uint64(101)), - // BlockSize: pointers.Ptr(uint64(102)), - // BufferSize: pointers.Ptr(uint64(103)), - // MaxBuffers: pointers.Ptr(uint64(104)), - // MaxBlocksPerFile: pointers.Ptr(uint64(105)), - // } - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, - // "--streaming=true", - // "--stream-cache-mb=101", - // "--block-size-mb=102", - // "--buffer-size-mb=103", - // "--max-buffers=104", - // "--max-blocks-per-file=105", - // "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) - // - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has disabled streaming", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - // Enabled: pointers.Ptr(false), - // StreamCache: pointers.Ptr(uint64(101)), - // BlockSize: pointers.Ptr(uint64(102)), - // BufferSize: pointers.Ptr(uint64(103)), - // MaxBuffers: pointers.Ptr(uint64(104)), - // MaxBlocksPerFile: pointers.Ptr(uint64(105)), - // } - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, - // "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) - // pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) - // return deploymentVolumesTestScenario{ - // name: "Do not change existing PersistentVolume with class name, when creating new PVC", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // pvcForAnotherComponent, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherComponent, - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) - // return deploymentVolumesTestScenario{ - // name: "Do not change existing PersistentVolume without class name, when creating new PVC", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // pvcForAnotherComponent, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherComponent, - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + type scenarioProperties struct { + changedNewRadixVolumeName string + changedNewRadixVolumeStorageName string + expectedVolumeName string + expectedNewSecretName string + expectedNewPvcName string + expectedNewPvName string + } + getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Update storage in existing volume name and storage", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.Name = scenarioProps.changedNewRadixVolumeName + vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) { + v.Name = scenarioProps.expectedVolumeName + }), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName + setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + changedNewRadixVolumeName: "volume101", + changedNewRadixVolumeStorageName: "storage101", + expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + expectedNewSecretName: "some-component-volume101-csiazurecreds", + expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + expectedNewPvName: "pv-radixvolumemount-some-uuid", + }), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) + pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) + pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + return deploymentVolumesTestScenario{ + name: "Garbage collect orphaned PVCs and PersistentVolume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createRandomPvc(props, props.namespace, props.componentName), + pvcForAnotherNamespace, + pvcForAnotherComponent, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherNamespace, + pvcForAnotherComponent, + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createRandomPv(props, props.namespace, props.componentName), + pvForAnotherNamespace, + pvForAnotherComponent, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + pvForAnotherNamespace, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Set readonly volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createRandomPvc(props, props.namespace, props.componentName), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createRandomPv(props, props.namespace, props.componentName), + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteOnce volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteMany volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadOnlyMany volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + pv.Spec.MountOptions = getMountOptions(props, true) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, + "--streaming=true", + "--stream-cache-mb=101", + "--block-size-mb=102", + "--buffer-size-mb=103", + "--max-buffers=104", + "--max-blocks-per-file=105", + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has disabled streaming", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + Enabled: pointers.Ptr(false), + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) + pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) + return deploymentVolumesTestScenario{ + name: "Do not change existing PersistentVolume with class name, when creating new PVC", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherComponent, + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + return deploymentVolumesTestScenario{ + name: "Do not change existing PersistentVolume without class name, when creating new PVC", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherComponent, + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) @@ -1014,15 +1011,15 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) testEnv := getTestEnv() deployment := getDeployment(testEnv) - deployment.radixDeployment = buildRd(appName, environment, componentName, scenario.radixVolumeMounts) - putExistingDeploymentVolumesScenarioDataToFakeCluster(&scenario, deployment) + radixDeployment := buildRd(appName, environment, componentName, scenario.radixVolumeMounts) + putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeclient, &scenario, deployment) desiredDeployment := getDesiredDeployment(componentName, scenario.volumes) - deployComponent := deployment.radixDeployment.Spec.Components[0] - err := createOrUpdateCsiAzureVolumeResources(context.Background(), testEnv.kubeUtil.KubeClient(), deployment.radixDeployment, environment, &deployComponent, desiredDeployment) + deployComponent := radixDeployment.Spec.Components[0] + err := createOrUpdateCsiAzureVolumeResources(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, environment, &deployComponent, desiredDeployment) assert.Nil(t, err) - existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment) + existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient(), deployment) assert.Nil(t, err) assert.True(t, equalPersistentVolumeClaims(&scenario.existingPvcsAfterTestRun, &existingPvcs), "PVC-s are not equal") assert.True(t, equalPersistentVolumes(&scenario.existingPVsAfterTestRun, &existingPvs), "PV-s are not equal") @@ -1141,23 +1138,23 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureKeyVaultResources( for _, scenario := range scenarios { testEnv := getTestEnv() deployment := getDeployment(testEnv) - deployment.radixDeployment = buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { + radixDeployment := buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { var builders []utils.DeployComponentBuilder builders = append(builders, utils.NewDeployComponentBuilder(). WithName(scenario.componentName). WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) return builders }) - radixDeployComponent := deployment.radixDeployment.GetComponentByName(scenario.componentName) + radixDeployComponent := radixDeployment.GetComponentByName(scenario.componentName) for _, azureKeyVault := range scenario.azureKeyVaults { - spc, err := deployment.createAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) + spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) if err != nil { t.Log(err.Error()) } else { t.Logf("created secret provider class %s", spc.Name) } } - volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, radixDeployComponent, deployment.radixDeployment.GetName(), nil) + volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, radixDeployComponent, radixDeployment.GetName(), nil) assert.Nil(t, err) assert.Len(t, volumes, len(scenario.expectedVolumeProps)) if len(scenario.expectedVolumeProps) == 0 { @@ -1189,23 +1186,23 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureKeyVaultResources( for _, scenario := range scenarios { testEnv := getTestEnv() deployment := getDeployment(testEnv) - deployment.radixDeployment = buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { + radixDeployment := buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { var builders []utils.DeployComponentBuilder builders = append(builders, utils.NewDeployComponentBuilder(). WithName(scenario.componentName). WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) return builders }) - radixDeployComponent := deployment.radixDeployment.GetComponentByName(scenario.componentName) + radixDeployComponent := radixDeployment.GetComponentByName(scenario.componentName) for _, azureKeyVault := range scenario.azureKeyVaults { - spc, err := deployment.createAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) + spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) if err != nil { t.Log(err.Error()) } else { t.Logf("created secret provider class %s", spc.Name) } } - volumeMounts, err := GetRadixDeployComponentVolumeMounts(radixDeployComponent, deployment.radixDeployment.GetName()) + volumeMounts, err := GetRadixDeployComponentVolumeMounts(radixDeployComponent, radixDeployment.GetName()) assert.Nil(t, err) assert.Len(t, volumeMounts, len(scenario.expectedVolumeProps)) if len(scenario.expectedVolumeProps) == 0 { @@ -1224,31 +1221,6 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureKeyVaultResources( }) } -func Test_EmptyDir(t *testing.T) { - appName, envName, compName := "anyapp", "anyenv", "anycomp" - - tu, kubeclient, kubeUtil, radixclient, kedaClient, prometheusclient, _, certClient := SetupTest(t) - builder := utils.NewDeploymentBuilder(). - WithRadixApplication(utils.NewRadixApplicationBuilder().WithAppName(appName).WithRadixRegistration(utils.NewRegistrationBuilder().WithName(appName))). - WithAppName(appName). - WithEnvironment(envName). - WithComponents( - utils.NewDeployComponentBuilder().WithName(compName).WithVolumeMounts( - v1.RadixVolumeMount{Name: "cache", Path: "/cache", EmptyDir: &v1.RadixEmptyDirVolumeMount{SizeLimit: resource.MustParse("50M")}}, - v1.RadixVolumeMount{Name: "log", Path: "/log", EmptyDir: &v1.RadixEmptyDirVolumeMount{SizeLimit: resource.MustParse("100M")}}, - ), - ) - - rd, err := ApplyDeploymentWithSync(tu, kubeclient, kubeUtil, radixclient, kedaClient, prometheusclient, certClient, builder) - require.NoError(t, err) - assert.NotNil(t, rd) - - deployment, err := kubeclient.AppsV1().Deployments(utils.GetEnvironmentNamespace(appName, envName)).Get(context.Background(), compName, metav1.GetOptions{}) - require.NoError(t, err) - assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 2) - assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2) -} - func createRandomPv(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolume { return createExpectedPv(props, func(pv *corev1.PersistentVolume) { pvName := getCsiAzurePersistentVolumeName() @@ -1351,26 +1323,26 @@ func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcPvProperties)) return props } -func putExistingDeploymentVolumesScenarioDataToFakeCluster(scenario *deploymentVolumesTestScenario, deployment *Deployment) { +func putExistingDeploymentVolumesScenarioDataToFakeCluster(kubeClient kubernetes.Interface, scenario *deploymentVolumesTestScenario, deployment *deployment.Deployment) { for _, pvc := range scenario.existingPvcsBeforeTestRun { - _, _ = deployment.kubeclient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.Background(), &pvc, metav1.CreateOptions{}) + _, _ = kubeClient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.Background(), &pvc, metav1.CreateOptions{}) } for _, pv := range scenario.existingPVsBeforeTestRun { - _, _ = deployment.kubeclient.CoreV1().PersistentVolumes().Create(context.Background(), &pv, metav1.CreateOptions{}) + _, _ = kubeClient.CoreV1().PersistentVolumes().Create(context.Background(), &pv, metav1.CreateOptions{}) } } -func getExistingPvcsAndPersistentVolumeFromFakeCluster(deployment *Deployment) ([]corev1.PersistentVolumeClaim, []corev1.PersistentVolume, error) { +func getExistingPvcsAndPersistentVolumeFromFakeCluster(kubeClient kubernetes.Interface, deployment *deployment.Deployment) ([]corev1.PersistentVolumeClaim, []corev1.PersistentVolume, error) { var pvcItems []corev1.PersistentVolumeClaim var pvItems []corev1.PersistentVolume - pvcList, err := deployment.kubeclient.CoreV1().PersistentVolumeClaims("").List(context.Background(), metav1.ListOptions{}) + pvcList, err := kubeClient.CoreV1().PersistentVolumeClaims("").List(context.Background(), metav1.ListOptions{}) if err != nil { return nil, nil, err } if pvcList != nil && pvcList.Items != nil { pvcItems = pvcList.Items } - pvList, err := deployment.kubeclient.CoreV1().PersistentVolumes().List(context.Background(), metav1.ListOptions{}) + pvList, err := kubeClient.CoreV1().PersistentVolumes().List(context.Background(), metav1.ListOptions{}) if err != nil { return nil, nil, err } @@ -1390,7 +1362,7 @@ func getDesiredDeployment(componentName string, volumes []corev1.Volume) *appsv1 Annotations: make(map[string]string), }, Spec: appsv1.DeploymentSpec{ - Replicas: pointers.Ptr(DefaultReplicas), + Replicas: pointers.Ptr(deployment.DefaultReplicas), Selector: &metav1.LabelSelector{MatchLabels: make(map[string]string)}, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: make(map[string]string), Annotations: make(map[string]string)}, From 143d5513583ef33a6ff699bae5e9a845826b7b8e Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 20 Dec 2024 14:04:40 +0100 Subject: [PATCH 37/68] Moving volumounts to own package --- pkg/apis/defaults/pod.go | 6 +- pkg/apis/deployment/azurekeyvault_test.go | 217 +++++++++++++++++ pkg/apis/deployment/deployment.go | 4 +- pkg/apis/deployment/deployment_test.go | 2 +- pkg/apis/deployment/kubedeployment.go | 12 +- pkg/apis/deployment/secrets.go | 2 +- .../deployment/serviceaccountspec_test.go | 10 +- pkg/apis/volumemount/volumemount.go | 216 ++++++++--------- pkg/apis/volumemount/volumemount_test.go | 220 +----------------- 9 files changed, 347 insertions(+), 342 deletions(-) create mode 100644 pkg/apis/deployment/azurekeyvault_test.go diff --git a/pkg/apis/defaults/pod.go b/pkg/apis/defaults/pod.go index 93a369c71..cddb421d2 100644 --- a/pkg/apis/defaults/pod.go +++ b/pkg/apis/defaults/pod.go @@ -1,6 +1,10 @@ package defaults const ( + // DefaultReplicas Hold the default replicas for the deployment if nothing is stated in the radix config + DefaultReplicas int32 = 1 + // DefaultNodeSelectorArchitecture Hold the default architecture for the deployment if nothing is stated in the radix config DefaultNodeSelectorArchitecture = "amd64" - DefaultNodeSelectorOS = "linux" + // DefaultNodeSelectorOS Hold the default OS for the deployment if nothing is stated in the radix config + DefaultNodeSelectorOS = "linux" ) diff --git a/pkg/apis/deployment/azurekeyvault_test.go b/pkg/apis/deployment/azurekeyvault_test.go new file mode 100644 index 000000000..dfe052e9f --- /dev/null +++ b/pkg/apis/deployment/azurekeyvault_test.go @@ -0,0 +1,217 @@ +package deployment + +import ( + "context" + "strings" + "testing" + + "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/volumemount" + "github.com/stretchr/testify/assert" +) + +func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { + appName := "app" + namespace := "some-namespace" + environment := "some-env" + componentName1, componentNameLong := "component1", "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit" + type expectedVolumeProps struct { + expectedVolumeNamePrefix string + expectedVolumeMountPath string + expectedNodePublishSecretRefName string + expectedVolumeAttributePrefixes map[string]string + } + scenarios := []struct { + name string + deployComponentBuilders []utils.DeployCommonComponentBuilder + componentName string + azureKeyVaults []v1.RadixAzureKeyVault + expectedVolumeProps []expectedVolumeProps + radixVolumeMounts []v1.RadixVolumeMount + }{ + { + name: "No Azure Key volumes as no RadixAzureKeyVault-s", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{}, + expectedVolumeProps: []expectedVolumeProps{}, + }, + { + name: "No Azure Key volumes as no secret names in secret object", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{{Name: "kv1"}}, + }, + { + name: "One Azure Key volume for one secret objects secret name", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{{ + Name: "kv1", + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + }}, + expectedVolumeProps: []expectedVolumeProps{ + { + expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", + expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", + expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "component1-az-keyvault-kv1-", + }, + }, + }, + }, + { + name: "Multiple Azure Key volumes for each RadixAzureKeyVault", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{ + { + Name: "kv1", + Path: utils.StringPtr("/mnt/customPath"), + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + }, + { + Name: "kv2", + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret2", EnvVar: "SECRET_REF2"}}, + }, + }, + expectedVolumeProps: []expectedVolumeProps{ + { + expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", + expectedVolumeMountPath: "/mnt/customPath", + expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "component1-az-keyvault-kv1-", + }, + }, + { + expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv2-", + expectedVolumeMountPath: "/mnt/azure-key-vault/kv2", + expectedNodePublishSecretRefName: "component1-kv2-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "component1-az-keyvault-kv2-", + }, + }, + }, + }, + { + name: "Volume name should be trimmed when exceeding 63 chars", + componentName: componentNameLong, + azureKeyVaults: []v1.RadixAzureKeyVault{{ + Name: "kv1", + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + }}, + expectedVolumeProps: []expectedVolumeProps{ + { + expectedVolumeNamePrefix: "a-very-long-component-name-that-exceeds-63-kubernetes-vol", + expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", + expectedNodePublishSecretRefName: "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-kv1-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-az-keyvault-kv1-", + }, + }, + }, + }, + } + t.Run("CSI Azure Key vault volumes", func(t *testing.T) { + t.Parallel() + for _, scenario := range scenarios { + deployment := getDeployment(t) + radixDeployment := buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { + var builders []utils.DeployComponentBuilder + builders = append(builders, utils.NewDeployComponentBuilder(). + WithName(scenario.componentName). + WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) + return builders + }) + radixDeployComponent := radixDeployment.GetComponentByName(scenario.componentName) + for _, azureKeyVault := range scenario.azureKeyVaults { + spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) + if err != nil { + t.Log(err.Error()) + } else { + t.Logf("created secret provider class %s", spc.Name) + } + } + volumes, err := volumemount.GetVolumes(context.Background(), deployment.kubeutil, namespace, radixDeployComponent, radixDeployment.GetName(), nil) + assert.Nil(t, err) + assert.Len(t, volumes, len(scenario.expectedVolumeProps)) + if len(scenario.expectedVolumeProps) == 0 { + continue + } + + for i := 0; i < len(volumes); i++ { + volume := volumes[i] + assert.Less(t, len(volume.Name), 64, "volume name is too long") + assert.NotNil(t, volume.CSI) + assert.NotNil(t, volume.CSI.VolumeAttributes) + assert.NotNil(t, volume.CSI.NodePublishSecretRef) + assert.Equal(t, "secrets-store.csi.k8s.io", volume.CSI.Driver) + + volumeProp := scenario.expectedVolumeProps[i] + for attrKey, attrValue := range volumeProp.expectedVolumeAttributePrefixes { + spcValue, exists := volume.CSI.VolumeAttributes[attrKey] + assert.True(t, exists) + assert.True(t, strings.HasPrefix(spcValue, attrValue)) + } + assert.True(t, strings.Contains(volume.Name, volumeProp.expectedVolumeNamePrefix)) + assert.Equal(t, volumeProp.expectedNodePublishSecretRefName, volume.CSI.NodePublishSecretRef.Name) + } + } + }) + + t.Run("CSI Azure Key vault volume mounts", func(t *testing.T) { + t.Parallel() + for _, scenario := range scenarios { + deployment := getDeployment(t) + radixDeployment := buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { + var builders []utils.DeployComponentBuilder + builders = append(builders, utils.NewDeployComponentBuilder(). + WithName(scenario.componentName). + WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) + return builders + }) + radixDeployComponent := radixDeployment.GetComponentByName(scenario.componentName) + for _, azureKeyVault := range scenario.azureKeyVaults { + spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) + if err != nil { + t.Log(err.Error()) + } else { + t.Logf("created secret provider class %s", spc.Name) + } + } + volumeMounts, err := volumemount.GetRadixDeployComponentVolumeMounts(radixDeployComponent, radixDeployment.GetName()) + assert.Nil(t, err) + assert.Len(t, volumeMounts, len(scenario.expectedVolumeProps)) + if len(scenario.expectedVolumeProps) == 0 { + continue + } + + for i := 0; i < len(volumeMounts); i++ { + volumeMount := volumeMounts[i] + volumeProp := scenario.expectedVolumeProps[i] + assert.Less(t, len(volumeMount.Name), 64, "volumemount name is too long") + assert.True(t, strings.Contains(volumeMount.Name, volumeProp.expectedVolumeNamePrefix)) + assert.Equal(t, volumeProp.expectedVolumeMountPath, volumeMount.MountPath) + assert.True(t, volumeMount.ReadOnly) + } + } + }) +} + +func getDeployment(t *testing.T) *Deployment { + tu, client, kubeUtil, radixClient, kedaClient, prometheusClient, _, certClient := SetupTest(t) + rd, _ := ApplyDeploymentWithSync(tu, client, kubeUtil, radixClient, kedaClient, prometheusClient, certClient, + utils.ARadixDeployment(). + WithComponents(utils.NewDeployComponentBuilder(). + WithName("comp1")). + WithAppName("any-app"). + WithEnvironment("test")) + return &Deployment{radixclient: radixClient, kubeutil: kubeUtil, radixDeployment: rd, config: &testConfig} +} + +func buildRdWithComponentBuilders(appName string, environment string, componentBuilders func() []utils.DeployComponentBuilder) *v1.RadixDeployment { + return utils.ARadixDeployment(). + WithAppName(appName). + WithEnvironment(environment). + WithComponents(componentBuilders()...). + BuildRD() +} diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index e0e2f37fd..7512d9fc1 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -31,9 +31,7 @@ import ( ) const ( - // DefaultReplicas Hold the default replicas for the deployment if nothing is stated in the radix config - DefaultReplicas int32 = 1 - prometheusInstanceLabel = "LABEL_PROMETHEUS_INSTANCE" + prometheusInstanceLabel = "LABEL_PROMETHEUS_INSTANCE" ) // DeploymentSyncer defines interface for syncing a RadixDeployment diff --git a/pkg/apis/deployment/deployment_test.go b/pkg/apis/deployment/deployment_test.go index 851178604..994028241 100644 --- a/pkg/apis/deployment/deployment_test.go +++ b/pkg/apis/deployment/deployment_test.go @@ -303,7 +303,7 @@ func TestObjectSynced_MultiComponent_ContainsAllElements(t *testing.T) { assert.True(t, deploymentByNameExists(componentNameRadixQuote, deployments), "radixquote deployment not there") spec := getDeploymentByName(componentNameRadixQuote, deployments).Spec - assert.Equal(t, DefaultReplicas, *spec.Replicas, "number of replicas was unexpected") + assert.Equal(t, defaults.DefaultReplicas, *spec.Replicas, "number of replicas was unexpected") assert.True(t, envVariableByNameExistOnDeployment(defaults.ContainerRegistryEnvironmentVariable, componentNameRadixQuote, deployments)) assert.True(t, envVariableByNameExistOnDeployment(defaults.RadixDNSZoneEnvironmentVariable, componentNameRadixQuote, deployments)) assert.True(t, envVariableByNameExistOnDeployment(defaults.ClusternameEnvironmentVariable, componentNameRadixQuote, deployments)) diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index 23f1b102a..225c1577f 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -39,7 +39,7 @@ func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployCompone return err } } - if err = volumemount.createOrUpdateCsiAzureVolumeResources(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace(), deployComponent, desiredDeployment); err != nil { + if err = volumemount.CreateOrUpdateCsiAzureVolumeResources(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace(), deployComponent, desiredDeployment); err != nil { return err } if err = deploy.handleJobAuxDeployment(ctx, deployComponent, desiredDeployment); err != nil { @@ -115,7 +115,7 @@ func (deploy *Deployment) getDesiredCreatedDeploymentConfig(ctx context.Context, desiredDeployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Labels: make(map[string]string), Annotations: make(map[string]string)}, Spec: appsv1.DeploymentSpec{ - Replicas: pointers.Ptr(DefaultReplicas), + Replicas: pointers.Ptr(defaults.DefaultReplicas), Selector: &metav1.LabelSelector{MatchLabels: make(map[string]string)}, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: make(map[string]string), Annotations: make(map[string]string)}, @@ -187,7 +187,7 @@ func (deploy *Deployment) getDesiredUpdatedDeploymentConfig(ctx context.Context, func (deploy *Deployment) getDeploymentPodLabels(deployComponent v1.RadixCommonDeployComponent) map[string]string { commitID := getDeployComponentCommitId(deployComponent) - labels := radixlabels.Merge( + lbs := radixlabels.Merge( radixlabels.ForApplicationName(deploy.radixDeployment.Spec.AppName), radixlabels.ForComponentName(deployComponent.GetName()), radixlabels.ForCommitId(commitID), @@ -195,10 +195,10 @@ func (deploy *Deployment) getDeploymentPodLabels(deployComponent v1.RadixCommonD ) if internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { - labels = radixlabels.Merge(labels, radixlabels.ForPodIsJobScheduler()) + lbs = radixlabels.Merge(lbs, radixlabels.ForPodIsJobScheduler()) } - return labels + return lbs } func (deploy *Deployment) getJobAuxDeploymentPodLabels(deployComponent v1.RadixCommonDeployComponent) map[string]string { @@ -354,7 +354,7 @@ func getDeployComponentReplicas(deployComponent v1.RadixCommonDeployComponent) i return int32(*override) } - componentReplicas := DefaultReplicas + componentReplicas := defaults.DefaultReplicas if replicas := deployComponent.GetReplicas(); replicas != nil { componentReplicas = int32(*replicas) } diff --git a/pkg/apis/deployment/secrets.go b/pkg/apis/deployment/secrets.go index 6062fb0f7..3bbb90075 100644 --- a/pkg/apis/deployment/secrets.go +++ b/pkg/apis/deployment/secrets.go @@ -69,7 +69,7 @@ func (deploy *Deployment) createOrUpdateSecretsForComponent(ctx context.Context, } secretsToManage = append(secretsToManage, volumeMountSecretsToManage...) - err = volumemount.garbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx, deploy.kubeutil, namespace, component, secretsToManage) + err = volumemount.GarbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx, deploy.kubeutil, namespace, component, secretsToManage) if err != nil { return err } diff --git a/pkg/apis/deployment/serviceaccountspec_test.go b/pkg/apis/deployment/serviceaccountspec_test.go index 1c32ba3c0..3d5cc1543 100644 --- a/pkg/apis/deployment/serviceaccountspec_test.go +++ b/pkg/apis/deployment/serviceaccountspec_test.go @@ -1,7 +1,7 @@ package deployment import ( - deployment2 "github.com/equinor/radix-operator/pkg/apis/internal/deployment" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "testing" "github.com/equinor/radix-common/utils/pointers" @@ -41,7 +41,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(false), spec.AutomountServiceAccountToken()) assert.Equal(t, utils.GetComponentServiceAccountName(rd.Spec.Components[1].Name), spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, deployment2.newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) + spec = NewServiceAccountSpec(rd, internal.NewJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) @@ -49,7 +49,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(false), spec.AutomountServiceAccountToken()) assert.Equal(t, defaultServiceAccountName, spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, deployment2.newJobSchedulerComponent(&rd.Spec.Jobs[1], rd)) + spec = NewServiceAccountSpec(rd, internal.NewJobSchedulerComponent(&rd.Spec.Jobs[1], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) @@ -72,7 +72,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixAPIServiceAccountName, spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, deployment2.newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) + spec = NewServiceAccountSpec(rd, internal.NewJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) @@ -96,7 +96,7 @@ func Test_ServiceAccountSpec(t *testing.T) { assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixGithubWebhookServiceAccountName, spec.ServiceAccountName()) - spec = NewServiceAccountSpec(rd, deployment2.newJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) + spec = NewServiceAccountSpec(rd, internal.NewJobSchedulerComponent(&rd.Spec.Jobs[0], rd)) assert.Equal(t, pointers.Ptr(true), spec.AutomountServiceAccountToken()) assert.Equal(t, defaults.RadixJobSchedulerServiceName, spec.ServiceAccountName()) diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index b72a6750e..c3daac5e7 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -47,15 +47,6 @@ var ( functionalPersistentVolumeClaimPhases = []corev1.PersistentVolumeClaimPhase{corev1.ClaimPending, corev1.ClaimBound} ) -// isKnownCsiAzureVolumeMount Supported volume mount type CSI Azure Blob volume -func isKnownCsiAzureVolumeMount(volumeMount string) bool { - switch volumeMount { - case string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure): - return true - } - return false -} - // GetRadixDeployComponentVolumeMounts Gets list of v1.VolumeMount for radixv1.RadixCommonDeployComponent func GetRadixDeployComponentVolumeMounts(deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.VolumeMount, error) { componentName := deployComponent.GetName() @@ -70,6 +61,103 @@ func GetRadixDeployComponentVolumeMounts(deployComponent radixv1.RadixCommonDepl return volumeMounts, nil } +// GetVolumes Get volumes of a component by RadixVolumeMounts +func GetVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { + var volumes []corev1.Volume + volumeMountVolumes, err := getComponentVolumeMountVolumes(deployComponent, existingVolumes) + if err != nil { + return nil, err + } + volumes = append(volumes, volumeMountVolumes...) + + storageRefsVolumes, err := getComponentSecretRefsVolumes(ctx, kubeutil, namespace, deployComponent, radixDeploymentName) + if err != nil { + return nil, err + } + volumes = append(volumes, storageRefsVolumes...) + + return volumes, nil +} + +// GarbageCollectVolumeMountsSecretsNoLongerInSpecForComponent Garbage collect volume-mount related secrets that are no longer in the spec +func GarbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx context.Context, kubeUtil *kube.Kube, namespace string, component radixv1.RadixCommonDeployComponent, excludeSecretNames []string) error { + secrets, err := listSecretsForVolumeMounts(ctx, kubeUtil, namespace, component) + if err != nil { + return err + } + for _, secret := range secrets { + if slice.Any(excludeSecretNames, func(s string) bool { return s == secret.Name }) { + continue + } + if err := kubeUtil.DeleteSecret(ctx, namespace, secret.Name); err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + + return garbageCollectSecrets(ctx, kubeUtil, namespace, secrets, excludeSecretNames) +} + +// CreateOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims, PersistentVolume +func CreateOrUpdateCsiAzureVolumeResources(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, deployComponent radixv1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) error { + componentName := deployComponent.GetName() + if err := createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx, kubeClient, radixDeployment, namespace, componentName, deployComponent.GetIdentity(), desiredDeployment); err != nil { + return err + } + currentlyUsedPvcNames, err := getCurrentlyUsedPersistentVolumeClaims(ctx, kubeClient, radixDeployment, desiredDeployment) + if err != nil { + return err + } + if err = garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx, kubeClient, namespace, componentName, currentlyUsedPvcNames); err != nil { + return err + } + return garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, kubeClient, currentlyUsedPvcNames) +} + +// CreateOrUpdateVolumeMountSecrets creates or updates secrets for volume mounts +func CreateOrUpdateVolumeMountSecrets(ctx context.Context, kubeUtil *kube.Kube, appName, namespace, componentName string, volumeMounts []radixv1.RadixVolumeMount) ([]string, error) { + var volumeMountSecretsToManage []string + for _, volumeMount := range volumeMounts { + secretName, accountKey, accountName := getCsiAzureVolumeMountCredsSecrets(ctx, kubeUtil, namespace, componentName, volumeMount.Name) + volumeMountSecretsToManage = append(volumeMountSecretsToManage, secretName) + err := createOrUpdateCsiAzureVolumeMountsSecrets(ctx, kubeUtil, appName, namespace, componentName, &volumeMount, secretName, accountName, accountKey) + if err != nil { + return nil, err + } + } + return volumeMountSecretsToManage, nil +} + +// GetCsiAzureVolumeMountType Gets the CSI Azure volume mount type +func GetCsiAzureVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) radixv1.MountType { + if radixVolumeMount.BlobFuse2 == nil { + return radixVolumeMount.Type + } + switch radixVolumeMount.BlobFuse2.Protocol { + case radixv1.BlobFuse2ProtocolFuse2, "": // default protocol if not set + return radixv1.MountTypeBlobFuse2Fuse2CsiAzure + default: + return "unsupported" + } +} + +func getCsiPersistentVolumesForNamespace(ctx context.Context, kubeClient kubernetes.Interface, namespace string, onlyFunctional bool) ([]corev1.PersistentVolume, error) { + pvList, err := kubeClient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + return slice.FindAll(pvList.Items, func(pv corev1.PersistentVolume) bool { + return pvIsForCsiDriver(pv) && pvIsForNamespace(pv, namespace) && (!onlyFunctional || pvIsFunctional(pv)) + }), nil +} + +func isKnownCsiAzureVolumeMount(volumeMount string) bool { + switch volumeMount { + case string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure): + return true + } + return false +} + func getRadixComponentVolumeMounts(deployComponent radixv1.RadixCommonDeployComponent) ([]corev1.VolumeMount, error) { if internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { return nil, nil @@ -116,8 +204,8 @@ func getCsiAzureKeyVaultSecretMountPath(azureKeyVault radixv1.RadixAzureKeyVault return *azureKeyVault.Path } -// volumeName: --- func getCsiAzureVolumeMountName(componentName string, volumeMount *radixv1.RadixVolumeMount) (string, error) { + // volumeName: --- csiVolumeType, err := getCsiRadixVolumeTypeId(volumeMount) if err != nil { return "", err @@ -135,19 +223,6 @@ func getCsiAzureVolumeMountName(componentName string, volumeMount *radixv1.Radix return trimVolumeNameToValidLength(fmt.Sprintf(csiVolumeNameTemplate, csiVolumeType, componentName, volumeMount.Name, csiAzureVolumeStorageName)), nil } -// GetCsiAzureVolumeMountType Gets the CSI Azure volume mount type -func GetCsiAzureVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) radixv1.MountType { - if radixVolumeMount.BlobFuse2 == nil { - return radixVolumeMount.Type - } - switch radixVolumeMount.BlobFuse2.Protocol { - case radixv1.BlobFuse2ProtocolFuse2, "": // default protocol if not set - return radixv1.MountTypeBlobFuse2Fuse2CsiAzure - default: - return "unsupported" - } -} - func getCsiRadixVolumeTypeId(radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { if radixVolumeMount.BlobFuse2 != nil { switch radixVolumeMount.BlobFuse2.Protocol { @@ -163,24 +238,6 @@ func getCsiRadixVolumeTypeId(radixVolumeMount *radixv1.RadixVolumeMount) (string return "", fmt.Errorf("unknown volume mount type %s", radixVolumeMount.Type) } -// GetVolumes Get volumes of a component by RadixVolumeMounts -func GetVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { - var volumes []corev1.Volume - volumeMountVolumes, err := getComponentVolumeMountVolumes(deployComponent, existingVolumes) - if err != nil { - return nil, err - } - volumes = append(volumes, volumeMountVolumes...) - - storageRefsVolumes, err := getComponentSecretRefsVolumes(ctx, kubeutil, namespace, deployComponent, radixDeploymentName) - if err != nil { - return nil, err - } - volumes = append(volumes, storageRefsVolumes...) - - return volumes, nil -} - func getComponentSecretRefsVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.Volume, error) { azureKeyVaultVolumes, err := getComponentSecretRefsAzureKeyVaultVolumes(ctx, kubeutil, namespace, deployComponent, radixDeploymentName) if err != nil { @@ -394,33 +451,6 @@ func createOrUpdateCsiAzureVolumeMountsSecrets(ctx context.Context, kubeUtil *ku return nil } -func garbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx context.Context, kubeUtil *kube.Kube, namespace string, component radixv1.RadixCommonDeployComponent, excludeSecretNames []string) error { - secrets, err := listSecretsForVolumeMounts(ctx, kubeUtil, namespace, component) - if err != nil { - return err - } - for _, secret := range secrets { - if slice.Any(excludeSecretNames, func(s string) bool { return s == secret.Name }) { - continue - } - if err := kubeUtil.DeleteSecret(ctx, namespace, secret.Name); err != nil && !k8serrors.IsNotFound(err) { - return err - } - } - - return garbageCollectSecrets(ctx, kubeUtil.KubeClient(), namespace, secrets, excludeSecretNames) -} - -func getCsiPersistentVolumesForNamespace(ctx context.Context, kubeClient kubernetes.Interface, namespace string, onlyFunctional bool) ([]corev1.PersistentVolume, error) { - pvList, err := kubeClient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) - if err != nil { - return nil, err - } - return slice.FindAll(pvList.Items, func(pv corev1.PersistentVolume) bool { - return pvIsForCsiDriver(pv) && pvIsForNamespace(pv, namespace) && (!onlyFunctional || pvIsFunctional(pv)) - }), nil -} - func pvIsForCsiDriver(pv corev1.PersistentVolume) bool { return pv.Spec.CSI != nil } @@ -512,9 +542,9 @@ func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, return persistentVolume } -// Specify a value the driver can use to uniquely identify the share in the cluster. -// https://github.com/kubernetes-csi/csi-driver-smb/blob/master/docs/driver-parameters.md#pvpvc-usage func getVolumeHandle(namespace, componentName, pvName, storageName string) string { + // Specify a value the driver can use to uniquely identify the share in the cluster. + // https://github.com/kubernetes-csi/csi-driver-smb/blob/master/docs/driver-parameters.md#pvpvc-usage return fmt.Sprintf("%s#%s#%s#%s", namespace, componentName, pvName, storageName) } @@ -754,22 +784,6 @@ func knownCSIDriver(driver string) bool { return ok } -// createOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims, PersistentVolume -func createOrUpdateCsiAzureVolumeResources(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, deployComponent radixv1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) error { - componentName := deployComponent.GetName() - if err := createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx, kubeClient, radixDeployment, namespace, componentName, deployComponent.GetIdentity(), desiredDeployment); err != nil { - return err - } - currentlyUsedPvcNames, err := getCurrentlyUsedPersistentVolumeClaims(ctx, kubeClient, radixDeployment, desiredDeployment) - if err != nil { - return err - } - if err = garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx, kubeClient, namespace, componentName, currentlyUsedPvcNames); err != nil { - return err - } - return garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, kubeClient, currentlyUsedPvcNames) -} - func createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace, componentName string, identity *radixv1.Identity, desiredDeployment *appsv1.Deployment) error { functionalPvList, err := getCsiPersistentVolumesForNamespace(ctx, kubeClient, namespace, true) if err != nil { @@ -970,20 +984,6 @@ func trimVolumeNameToValidLength(volumeName string) string { return fmt.Sprintf("%s-%s", volumeName[:63-randSize-1], randString) } -// CreateOrUpdateVolumeMountSecrets creates or updates secrets for volume mounts -func CreateOrUpdateVolumeMountSecrets(ctx context.Context, kubeUtil *kube.Kube, appName, namespace, componentName string, volumeMounts []radixv1.RadixVolumeMount) ([]string, error) { - var volumeMountSecretsToManage []string - for _, volumeMount := range volumeMounts { - secretName, accountKey, accountName := getCsiAzureVolumeMountCredsSecrets(ctx, kubeUtil, namespace, componentName, volumeMount.Name) - volumeMountSecretsToManage = append(volumeMountSecretsToManage, secretName) - err := createOrUpdateCsiAzureVolumeMountsSecrets(ctx, kubeUtil, appName, namespace, componentName, &volumeMount, secretName, accountName, accountKey) - if err != nil { - return nil, err - } - } - return volumeMountSecretsToManage, nil -} - func getCsiAzureVolumeMountCredsSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace, componentName, volumeMountName string) (string, []byte, []byte) { secretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, volumeMountName) accountKey := []byte(defaults.SecretDefaultData) @@ -1009,26 +1009,14 @@ func listSecretsForVolumeMounts(ctx context.Context, kubeUtil *kube.Kube, namesp return csiSecrets, nil } -// garbageCollectSecrets delete secrets, excluding with names in the excludeSecretNames -func garbageCollectSecrets(ctx context.Context, kubeClient kubernetes.Interface, namespace string, secrets []*corev1.Secret, excludeSecretNames []string) error { +func garbageCollectSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace string, secrets []*corev1.Secret, excludeSecretNames []string) error { for _, secret := range secrets { if slice.Any(excludeSecretNames, func(s string) bool { return s == secret.Name }) { continue } - err := DeleteSecret(ctx, kubeClient, namespace, secret) - if err != nil { + if err := kubeUtil.DeleteSecret(ctx, namespace, secret.GetName()); err != nil && !k8serrors.IsNotFound(err) { return err } } return nil } - -func DeleteSecret(ctx context.Context, kubeClient kubernetes.Interface, namespace string, secret *corev1.Secret) error { - log.Ctx(ctx).Debug().Msgf("Delete secret %s", secret.Name) - err := kubeClient.CoreV1().Secrets(namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - log.Ctx(ctx).Info().Msgf("Deleted secret: %s in namespace %s", secret.GetName(), namespace) - return nil -} diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index f7a05f535..adae4d6ae 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "fmt" - "github.com/equinor/radix-operator/pkg/apis/deployment" + "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/google/uuid" "k8s.io/apimachinery/pkg/types" "strings" @@ -13,7 +13,6 @@ import ( commonUtils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/pointers" - "github.com/equinor/radix-operator/pkg/apis/config" "github.com/equinor/radix-operator/pkg/apis/internal/persistentvolume" "github.com/equinor/radix-operator/pkg/apis/kube" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" @@ -102,16 +101,6 @@ func getTestEnv() TestEnv { return testEnv } -func getDeployment(testEnv TestEnv) *deployment.Deployment { - return &deployment.Deployment{ - kubeclient: testEnv.kubeclient, - radixclient: testEnv.radixclient, - kubeutil: testEnv.kubeUtil, - prometheusperatorclient: testEnv.prometheusclient, - config: &config.Config{}, - } -} - func (suite *VolumeMountTestSuite) Test_NoVolumeMounts() { suite.T().Run("app", func(t *testing.T) { t.Parallel() @@ -373,7 +362,6 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { suite.T().Run("No volumes", func(t *testing.T) { t.Parallel() testEnv := getTestEnv() - deployment := getDeployment(testEnv) for _, factory := range suite.radixCommonDeployComponentFactories { t.Logf("Test case for component %s", factory.GetTargetType()) @@ -389,7 +377,6 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { suite.T().Run("Exists volume", func(t *testing.T) { t.Parallel() testEnv := getTestEnv() - deployment := getDeployment(testEnv) for _, factory := range suite.radixCommonDeployComponentFactories { for _, scenario := range scenarios { t.Logf("Test case %s for component %s", scenario.name, factory.GetTargetType()) @@ -1010,16 +997,15 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { for _, scenario := range scenarios { t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) testEnv := getTestEnv() - deployment := getDeployment(testEnv) radixDeployment := buildRd(appName, environment, componentName, scenario.radixVolumeMounts) - putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeclient, &scenario, deployment) + putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeclient, &scenario) desiredDeployment := getDesiredDeployment(componentName, scenario.volumes) deployComponent := radixDeployment.Spec.Components[0] - err := createOrUpdateCsiAzureVolumeResources(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, environment, &deployComponent, desiredDeployment) + err := CreateOrUpdateCsiAzureVolumeResources(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, environment, &deployComponent, desiredDeployment) assert.Nil(t, err) - existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient(), deployment) + existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) assert.Nil(t, err) assert.True(t, equalPersistentVolumeClaims(&scenario.existingPvcsAfterTestRun, &existingPvcs), "PVC-s are not equal") assert.True(t, equalPersistentVolumes(&scenario.existingPVsAfterTestRun, &existingPvs), "PV-s are not equal") @@ -1033,194 +1019,6 @@ func matchPvAndPvc(pv *corev1.PersistentVolume, pvc *corev1.PersistentVolumeClai pvc.Spec.VolumeName = pv.Name } -func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureKeyVaultResources() { - appName := "app" - namespace := "some-namespace" - environment := "some-env" - componentName1, componentNameLong := "component1", "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit" - type expectedVolumeProps struct { - expectedVolumeNamePrefix string - expectedVolumeMountPath string - expectedNodePublishSecretRefName string - expectedVolumeAttributePrefixes map[string]string - } - scenarios := []struct { - name string - deployComponentBuilders []utils.DeployCommonComponentBuilder - componentName string - azureKeyVaults []v1.RadixAzureKeyVault - expectedVolumeProps []expectedVolumeProps - radixVolumeMounts []v1.RadixVolumeMount - }{ - { - name: "No Azure Key volumes as no RadixAzureKeyVault-s", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{}, - expectedVolumeProps: []expectedVolumeProps{}, - }, - { - name: "No Azure Key volumes as no secret names in secret object", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{{Name: "kv1"}}, - }, - { - name: "One Azure Key volume for one secret objects secret name", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{{ - Name: "kv1", - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - }}, - expectedVolumeProps: []expectedVolumeProps{ - { - expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", - expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", - expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "component1-az-keyvault-kv1-", - }, - }, - }, - }, - { - name: "Multiple Azure Key volumes for each RadixAzureKeyVault", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{ - { - Name: "kv1", - Path: utils.StringPtr("/mnt/customPath"), - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - }, - { - Name: "kv2", - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret2", EnvVar: "SECRET_REF2"}}, - }, - }, - expectedVolumeProps: []expectedVolumeProps{ - { - expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", - expectedVolumeMountPath: "/mnt/customPath", - expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "component1-az-keyvault-kv1-", - }, - }, - { - expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv2-", - expectedVolumeMountPath: "/mnt/azure-key-vault/kv2", - expectedNodePublishSecretRefName: "component1-kv2-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "component1-az-keyvault-kv2-", - }, - }, - }, - }, - { - name: "Volume name should be trimmed when exceeding 63 chars", - componentName: componentNameLong, - azureKeyVaults: []v1.RadixAzureKeyVault{{ - Name: "kv1", - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - }}, - expectedVolumeProps: []expectedVolumeProps{ - { - expectedVolumeNamePrefix: "a-very-long-component-name-that-exceeds-63-kubernetes-vol", - expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", - expectedNodePublishSecretRefName: "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-kv1-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-az-keyvault-kv1-", - }, - }, - }, - }, - } - suite.T().Run("CSI Azure Key vault volumes", func(t *testing.T) { - t.Parallel() - for _, scenario := range scenarios { - testEnv := getTestEnv() - deployment := getDeployment(testEnv) - radixDeployment := buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { - var builders []utils.DeployComponentBuilder - builders = append(builders, utils.NewDeployComponentBuilder(). - WithName(scenario.componentName). - WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) - return builders - }) - radixDeployComponent := radixDeployment.GetComponentByName(scenario.componentName) - for _, azureKeyVault := range scenario.azureKeyVaults { - spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) - if err != nil { - t.Log(err.Error()) - } else { - t.Logf("created secret provider class %s", spc.Name) - } - } - volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, namespace, radixDeployComponent, radixDeployment.GetName(), nil) - assert.Nil(t, err) - assert.Len(t, volumes, len(scenario.expectedVolumeProps)) - if len(scenario.expectedVolumeProps) == 0 { - continue - } - - for i := 0; i < len(volumes); i++ { - volume := volumes[i] - assert.Less(t, len(volume.Name), 64, "volume name is too long") - assert.NotNil(t, volume.CSI) - assert.NotNil(t, volume.CSI.VolumeAttributes) - assert.NotNil(t, volume.CSI.NodePublishSecretRef) - assert.Equal(t, "secrets-store.csi.k8s.io", volume.CSI.Driver) - - volumeProp := scenario.expectedVolumeProps[i] - for attrKey, attrValue := range volumeProp.expectedVolumeAttributePrefixes { - spcValue, exists := volume.CSI.VolumeAttributes[attrKey] - assert.True(t, exists) - assert.True(t, strings.HasPrefix(spcValue, attrValue)) - } - assert.True(t, strings.Contains(volume.Name, volumeProp.expectedVolumeNamePrefix)) - assert.Equal(t, volumeProp.expectedNodePublishSecretRefName, volume.CSI.NodePublishSecretRef.Name) - } - } - }) - - suite.T().Run("CSI Azure Key vault volume mounts", func(t *testing.T) { - t.Parallel() - for _, scenario := range scenarios { - testEnv := getTestEnv() - deployment := getDeployment(testEnv) - radixDeployment := buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { - var builders []utils.DeployComponentBuilder - builders = append(builders, utils.NewDeployComponentBuilder(). - WithName(scenario.componentName). - WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) - return builders - }) - radixDeployComponent := radixDeployment.GetComponentByName(scenario.componentName) - for _, azureKeyVault := range scenario.azureKeyVaults { - spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) - if err != nil { - t.Log(err.Error()) - } else { - t.Logf("created secret provider class %s", spc.Name) - } - } - volumeMounts, err := GetRadixDeployComponentVolumeMounts(radixDeployComponent, radixDeployment.GetName()) - assert.Nil(t, err) - assert.Len(t, volumeMounts, len(scenario.expectedVolumeProps)) - if len(scenario.expectedVolumeProps) == 0 { - continue - } - - for i := 0; i < len(volumeMounts); i++ { - volumeMount := volumeMounts[i] - volumeProp := scenario.expectedVolumeProps[i] - assert.Less(t, len(volumeMount.Name), 64, "volumemount name is too long") - assert.True(t, strings.Contains(volumeMount.Name, volumeProp.expectedVolumeNamePrefix)) - assert.Equal(t, volumeProp.expectedVolumeMountPath, volumeMount.MountPath) - assert.True(t, volumeMount.ReadOnly) - } - } - }) -} - func createRandomPv(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolume { return createExpectedPv(props, func(pv *corev1.PersistentVolume) { pvName := getCsiAzurePersistentVolumeName() @@ -1323,7 +1121,7 @@ func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcPvProperties)) return props } -func putExistingDeploymentVolumesScenarioDataToFakeCluster(kubeClient kubernetes.Interface, scenario *deploymentVolumesTestScenario, deployment *deployment.Deployment) { +func putExistingDeploymentVolumesScenarioDataToFakeCluster(kubeClient kubernetes.Interface, scenario *deploymentVolumesTestScenario) { for _, pvc := range scenario.existingPvcsBeforeTestRun { _, _ = kubeClient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.Background(), &pvc, metav1.CreateOptions{}) } @@ -1332,7 +1130,7 @@ func putExistingDeploymentVolumesScenarioDataToFakeCluster(kubeClient kubernetes } } -func getExistingPvcsAndPersistentVolumeFromFakeCluster(kubeClient kubernetes.Interface, deployment *deployment.Deployment) ([]corev1.PersistentVolumeClaim, []corev1.PersistentVolume, error) { +func getExistingPvcsAndPersistentVolumeFromFakeCluster(kubeClient kubernetes.Interface) ([]corev1.PersistentVolumeClaim, []corev1.PersistentVolume, error) { var pvcItems []corev1.PersistentVolumeClaim var pvItems []corev1.PersistentVolume pvcList, err := kubeClient.CoreV1().PersistentVolumeClaims("").List(context.Background(), metav1.ListOptions{}) @@ -1362,7 +1160,7 @@ func getDesiredDeployment(componentName string, volumes []corev1.Volume) *appsv1 Annotations: make(map[string]string), }, Spec: appsv1.DeploymentSpec{ - Replicas: pointers.Ptr(deployment.DefaultReplicas), + Replicas: pointers.Ptr(defaults.DefaultReplicas), Selector: &metav1.LabelSelector{MatchLabels: make(map[string]string)}, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: make(map[string]string), Annotations: make(map[string]string)}, @@ -1624,8 +1422,8 @@ func createExpectedAutoProvisionedPvcWithStorageClass(props expectedPvcPvPropert annotations := map[string]string{ "pv.kubernetes.io/bind-completed": "yes", "pv.kubernetes.io/bound-by-controller": "yes", - "volume.beta.kubernetes.io/storage-provisioner": "blob.csi.azure.com", - "volume.kubernetes.io/storage-provisioner": "blob.csi.azure.com", + "volume.beta.kubernetes.io/storage-provisioner": provisionerBlobCsiAzure, + "volume.kubernetes.io/storage-provisioner": provisionerBlobCsiAzure, } pvc := corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ From f0c4ae0cc6c2872184c26ecb36a54989751be7cc Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 20 Dec 2024 16:47:04 +0100 Subject: [PATCH 38/68] Fixing volumes for jobs --- pkg/apis/batch/kubejob.go | 42 +++++++++--------------- pkg/apis/batch/syncer.go | 18 +++++++--- pkg/apis/deployment/deployment.go | 17 ++++------ pkg/apis/deployment/kubedeployment.go | 18 +++++++--- pkg/apis/volumemount/volumemount.go | 40 +++++++++++----------- pkg/apis/volumemount/volumemount_test.go | 5 +-- 6 files changed, 73 insertions(+), 67 deletions(-) diff --git a/pkg/apis/batch/kubejob.go b/pkg/apis/batch/kubejob.go index c48e23557..c0321ffc1 100644 --- a/pkg/apis/batch/kubejob.go +++ b/pkg/apis/batch/kubejob.go @@ -28,7 +28,7 @@ const ( jobPayloadVolumeName = "job-payload" ) -func (s *syncer) reconcileKubeJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, rd *radixv1.RadixDeployment, jobComponent *radixv1.RadixDeployJobComponent, existingJobs []*batchv1.Job) error { +func (s *syncer) reconcileKubeJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, rd *radixv1.RadixDeployment, jobComponent *radixv1.RadixDeployJobComponent, existingJobs []*batchv1.Job, volumes []corev1.Volume) error { batchJobKubeJobs := slice.FindAll(existingJobs, isKubeJobForBatchJob(batchJob)) if isBatchJobStopRequested(batchJob) { @@ -52,7 +52,7 @@ func (s *syncer) reconcileKubeJob(ctx context.Context, batchJob *radixv1.RadixBa return err } - job, err := s.buildJob(ctx, batchJob, jobComponent, rd) + job, err := s.buildJob(ctx, batchJob, jobComponent, rd, volumes) if err != nil { return err } @@ -93,7 +93,7 @@ func (s *syncer) deleteJobs(ctx context.Context, jobsToDelete []*batchv1.Job) er return nil } -func (s *syncer) buildJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, jobComponent *radixv1.RadixDeployJobComponent, rd *radixv1.RadixDeployment) (*batchv1.Job, error) { +func (s *syncer) buildJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, jobComponent *radixv1.RadixDeployJobComponent, rd *radixv1.RadixDeployment, volumes []corev1.Volume) (*batchv1.Job, error) { jobLabels := s.batchJobIdentifierLabel(batchJob.Name, rd.Spec.AppName) podLabels := radixlabels.Merge( jobLabels, @@ -101,11 +101,6 @@ func (s *syncer) buildJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, ) podAnnotations := annotations.ForClusterAutoscalerSafeToEvict(false) - volumes, err := s.getVolumes(ctx, rd.GetNamespace(), rd.Spec.Environment, batchJob, jobComponent, rd.Name) - if err != nil { - return nil, err - } - kubeJobName := getKubeJobName(s.radixBatch.GetName(), batchJob.Name) containers, err := s.getContainers(ctx, rd, jobComponent, batchJob, kubeJobName) if err != nil { @@ -136,6 +131,7 @@ func (s *syncer) buildJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, } serviceAccountSpec := deployment.NewServiceAccountSpec(rd, jobComponent) + volumes = s.appendPayloadSecretVolumes(batchJob, jobComponent, volumes) job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -180,27 +176,21 @@ func (s *syncer) getJobPodImagePullSecrets(rd *radixv1.RadixDeployment) []corev1 return imagePullSecrets } -func (s *syncer) getVolumes(ctx context.Context, namespace, environment string, batchJob *radixv1.RadixBatchJob, radixJobComponent *radixv1.RadixDeployJobComponent, radixDeploymentName string) ([]corev1.Volume, error) { - volumes, err := volumemount.GetVolumes(ctx, s.kubeUtil, namespace, radixJobComponent, radixDeploymentName, nil) - if err != nil { - return nil, err +func (s *syncer) appendPayloadSecretVolumes(batchJob *radixv1.RadixBatchJob, radixJobComponent *radixv1.RadixDeployJobComponent, volumes []corev1.Volume) []corev1.Volume { + if radixJobComponent.Payload == nil || batchJob.PayloadSecretRef == nil { + return nil } - - if radixJobComponent.Payload != nil && batchJob.PayloadSecretRef != nil { - volumes = append(volumes, corev1.Volume{ - Name: jobPayloadVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: batchJob.PayloadSecretRef.Name, - Items: []corev1.KeyToPath{ - {Key: batchJob.PayloadSecretRef.Key, Path: "payload"}, - }, + return append(volumes, corev1.Volume{ + Name: jobPayloadVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: batchJob.PayloadSecretRef.Name, + Items: []corev1.KeyToPath{ + {Key: batchJob.PayloadSecretRef.Key, Path: "payload"}, }, }, - }) - } - - return volumes, nil + }, + }) } func (s *syncer) getContainers(ctx context.Context, rd *radixv1.RadixDeployment, jobComponent *radixv1.RadixDeployJobComponent, batchJob *radixv1.RadixBatchJob, kubeJobName string) ([]corev1.Container, error) { diff --git a/pkg/apis/batch/syncer.go b/pkg/apis/batch/syncer.go index 5febda8eb..25a7b84d4 100644 --- a/pkg/apis/batch/syncer.go +++ b/pkg/apis/batch/syncer.go @@ -3,13 +3,13 @@ package batch import ( "context" "fmt" - commonutils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/slice" "github.com/equinor/radix-operator/pkg/apis/config" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" + "github.com/equinor/radix-operator/pkg/apis/volumemount" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" "github.com/rs/zerolog/log" "k8s.io/apimachinery/pkg/api/errors" @@ -86,22 +86,32 @@ func (s *syncer) reconcile(ctx context.Context) error { return err } - existingJobs, err := s.kubeUtil.ListJobsWithSelector(ctx, s.radixBatch.GetNamespace(), s.batchIdentifierLabel().String()) + namespace := s.radixBatch.GetNamespace() + existingJobs, err := s.kubeUtil.ListJobsWithSelector(ctx, namespace, s.batchIdentifierLabel().String()) if err != nil { return err } - existingServices, err := s.kubeUtil.ListServicesWithSelector(ctx, s.radixBatch.GetNamespace(), s.batchIdentifierLabel().String()) + existingServices, err := s.kubeUtil.ListServicesWithSelector(ctx, namespace, s.batchIdentifierLabel().String()) if err != nil { return err } + desiredVolumes, err := volumemount.GetVolumes(ctx, s.kubeUtil, namespace, jobComponent, rd.Name, nil) + if err != nil { + return err + } + actualVolumes, err := volumemount.CreateOrUpdateCsiAzureVolumeResources(ctx, s.kubeUtil.KubeClient(), rd, namespace, jobComponent, desiredVolumes) + if err != nil { + return fmt.Errorf("failed to create or update csi azure volume resources: %w", err) + } + for i, batchJob := range s.radixBatch.Spec.Jobs { if err := s.reconcileService(ctx, &batchJob, rd, jobComponent, existingServices); err != nil { return fmt.Errorf("batchjob %s: failed to reconcile service: %w", batchJob.Name, err) } - if err := s.reconcileKubeJob(ctx, &batchJob, rd, jobComponent, existingJobs); err != nil { + if err := s.reconcileKubeJob(ctx, &batchJob, rd, jobComponent, existingJobs, actualVolumes); err != nil { return fmt.Errorf("batchjob %s: failed to reconcile kubejob: %w", batchJob.Name, err) } diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 7512d9fc1..94a7b9c76 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -588,8 +588,8 @@ func (deploy *Deployment) syncDeploymentForRadixComponent(ctx context.Context, c return nil } -func (deploy *Deployment) createOrUpdateJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) (*appsv1.Deployment, *appsv1.Deployment, error) { - currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.getCurrentAndDesiredJobAuxDeployment(ctx, deployComponent, desiredDeployment) +func (deploy *Deployment) createOrUpdateJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, namespace, deploymentName string, volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) (*appsv1.Deployment, *appsv1.Deployment, error) { + currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.getCurrentAndDesiredJobAuxDeployment(ctx, deployComponent, namespace, deploymentName) if err != nil { return nil, nil, err } @@ -598,12 +598,9 @@ func (deploy *Deployment) createOrUpdateJobAuxDeployment(ctx context.Context, de desiredJobAuxDeployment.Spec.Template.Labels = deploy.getJobAuxDeploymentPodLabels(deployComponent) desiredJobAuxDeployment.Spec.Template.Spec.ServiceAccountName = (&radixComponentServiceAccountSpec{component: deployComponent}).ServiceAccountName() desiredJobAuxDeployment.Spec.Template.Spec.Affinity = utils.GetAffinityForJobAPIAuxComponent() - // Copy volumes and volume mounts from desired deployment to job aux deployment - desiredJobAuxDeployment.Spec.Template.Spec.Volumes = desiredDeployment.Spec.Template.Spec.Volumes - desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts - // Remove volumes and volume mounts from job scheduler deployment - desiredDeployment.Spec.Template.Spec.Volumes = nil - desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = nil + // Move volumes and volume mounts from desired deployment to job aux deployment + desiredJobAuxDeployment.Spec.Template.Spec.Volumes = volumes + desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts syncRadixRestartEnvironmentVariable(deployComponent, desiredJobAuxDeployment) return currentJobAuxDeployment, desiredJobAuxDeployment, nil @@ -622,8 +619,8 @@ func syncRadixRestartEnvironmentVariable(deployComponent v1.RadixCommonDeployCom } } -func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) (*appsv1.Deployment, *appsv1.Deployment, error) { - currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(ctx, desiredDeployment.Namespace, getJobAuxObjectName(desiredDeployment.Name)) +func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, namespace, deploymentName string) (*appsv1.Deployment, *appsv1.Deployment, error) { + currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(ctx, namespace, deploymentName) if err != nil { if k8sErrors.IsNotFound(err) { return nil, deploy.createJobAuxDeployment(deployComponent), nil diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index 225c1577f..82dfd9ae8 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -39,20 +39,25 @@ func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployCompone return err } } - if err = volumemount.CreateOrUpdateCsiAzureVolumeResources(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace(), deployComponent, desiredDeployment); err != nil { + actualVolumes, err := volumemount.CreateOrUpdateCsiAzureVolumeResources(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace(), deployComponent, desiredDeployment.Spec.Template.Spec.Volumes) + if err != nil { return err } - if err = deploy.handleJobAuxDeployment(ctx, deployComponent, desiredDeployment); err != nil { + desiredDeployment.Spec.Template.Spec.Volumes = actualVolumes + desiredVolumeMounts := desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts + if err = deploy.handleJobAuxDeployment(ctx, deployComponent, desiredDeployment, actualVolumes, desiredVolumeMounts); err != nil { return err } return deploy.kubeutil.ApplyDeployment(ctx, deploy.radixDeployment.Namespace, currentDeployment, desiredDeployment) } -func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) error { +func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment, volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) error { if !internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { return nil } - currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, desiredDeployment) + namespace := desiredDeployment.GetNamespace() + deploymentName := desiredDeployment.GetName() + currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, namespace, deploymentName, volumes, volumeMounts) if err != nil { return err } @@ -65,11 +70,14 @@ func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, deployComp return err } - currentJobAuxDeployment, desiredJobAuxDeployment, err = deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, desiredDeployment) + currentJobAuxDeployment, desiredJobAuxDeployment, err = deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, namespace, deploymentName, volumes, volumeMounts) if err != nil { return err } } + // Remove volumes and volume mounts from job scheduler deployment, they are set to aux deployment + desiredDeployment.Spec.Template.Spec.Volumes = nil + desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = nil return deploy.kubeutil.ApplyDeployment(ctx, deploy.radixDeployment.Namespace, currentJobAuxDeployment, desiredJobAuxDeployment) } diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index c3daac5e7..c0e1e2c7b 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -18,7 +18,6 @@ import ( radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/google/uuid" "github.com/rs/zerolog/log" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -62,7 +61,7 @@ func GetRadixDeployComponentVolumeMounts(deployComponent radixv1.RadixCommonDepl } // GetVolumes Get volumes of a component by RadixVolumeMounts -func GetVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { +func GetVolumes(ctx context.Context, kubeUtil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string, existingVolumes []corev1.Volume) ([]corev1.Volume, error) { var volumes []corev1.Volume volumeMountVolumes, err := getComponentVolumeMountVolumes(deployComponent, existingVolumes) if err != nil { @@ -70,7 +69,7 @@ func GetVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, depl } volumes = append(volumes, volumeMountVolumes...) - storageRefsVolumes, err := getComponentSecretRefsVolumes(ctx, kubeutil, namespace, deployComponent, radixDeploymentName) + storageRefsVolumes, err := getComponentSecretRefsVolumes(ctx, kubeUtil, namespace, deployComponent, radixDeploymentName) if err != nil { return nil, err } @@ -98,19 +97,21 @@ func GarbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx context.Con } // CreateOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims, PersistentVolume -func CreateOrUpdateCsiAzureVolumeResources(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, deployComponent radixv1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) error { +// Returns actual volumes, with existing relevant PersistentVolumeClaimName and PersistentVolumeName +func CreateOrUpdateCsiAzureVolumeResources(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, deployComponent radixv1.RadixCommonDeployComponent, desiredVolumes []corev1.Volume) ([]corev1.Volume, error) { componentName := deployComponent.GetName() - if err := createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx, kubeClient, radixDeployment, namespace, componentName, deployComponent.GetIdentity(), desiredDeployment); err != nil { - return err + actualVolumes, err := createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx, kubeClient, radixDeployment, namespace, componentName, deployComponent.GetIdentity(), desiredVolumes) + if err != nil { + return nil, err } - currentlyUsedPvcNames, err := getCurrentlyUsedPersistentVolumeClaims(ctx, kubeClient, radixDeployment, desiredDeployment) + currentlyUsedPvcNames, err := getCurrentlyUsedPersistentVolumeClaims(ctx, kubeClient, radixDeployment, actualVolumes) if err != nil { - return err + return nil, err } if err = garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx, kubeClient, namespace, componentName, currentlyUsedPvcNames); err != nil { - return err + return nil, err } - return garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, kubeClient, currentlyUsedPvcNames) + return actualVolumes, garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, kubeClient, currentlyUsedPvcNames) } // CreateOrUpdateVolumeMountSecrets creates or updates secrets for volume mounts @@ -784,19 +785,19 @@ func knownCSIDriver(driver string) bool { return ok } -func createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace, componentName string, identity *radixv1.Identity, desiredDeployment *appsv1.Deployment) error { +func createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace, componentName string, identity *radixv1.Identity, desiredVolumes []corev1.Volume) ([]corev1.Volume, error) { functionalPvList, err := getCsiPersistentVolumesForNamespace(ctx, kubeClient, namespace, true) if err != nil { - return err + return nil, err } pvcByNameMap, err := getPvcByNameMap(ctx, kubeClient, namespace, componentName) if err != nil { - return err + return nil, err } radixVolumeMountsByNameMap := getRadixVolumeMountsByNameMap(radixDeployment, componentName) var errs []error var volumes []corev1.Volume - for _, volume := range desiredDeployment.Spec.Template.Spec.Volumes { + for _, volume := range desiredVolumes { if volume.PersistentVolumeClaim == nil { volumes = append(volumes, volume) continue @@ -815,10 +816,9 @@ func createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, kubeCl } } if len(errs) > 0 { - return errors.Join(errs...) + return nil, errors.Join(errs...) } - desiredDeployment.Spec.Template.Spec.Volumes = volumes - return nil + return volumes, nil } func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, componentName string, identity *radixv1.Identity, volume corev1.Volume, radixVolumeMount *radixv1.RadixVolumeMount, functionalPvList []corev1.PersistentVolume, pvcByNameMap map[string]*corev1.PersistentVolumeClaim) (*corev1.Volume, error) { @@ -865,7 +865,7 @@ func getPvcByNameMap(ctx context.Context, kubeClient kubernetes.Interface, names return persistentvolume.GetPersistentVolumeClaimMap(&pvcList.Items), nil } -func getCurrentlyUsedPersistentVolumeClaims(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, desiredDeployment *appsv1.Deployment) (map[string]any, error) { +func getCurrentlyUsedPersistentVolumeClaims(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, volumes []corev1.Volume) (map[string]any, error) { namespace := radixDeployment.GetNamespace() pvcNames := make(map[string]any) deploymentList, err := kubeClient.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) @@ -882,8 +882,8 @@ func getCurrentlyUsedPersistentVolumeClaims(ctx context.Context, kubeClient kube for _, job := range jobsList.Items { pvcNames = appendUsedPersistenceVolumeClaimsFrom(pvcNames, job.Spec.Template.Spec.Volumes) } - // TODD add from RadixDeployments, connected to existing scheduled jobs and batches - return appendUsedPersistenceVolumeClaimsFrom(pvcNames, desiredDeployment.Spec.Template.Spec.Volumes), nil + // TODO add from RadixDeployments, connected to existing scheduled jobs and batches + return appendUsedPersistenceVolumeClaimsFrom(pvcNames, volumes), nil } func appendUsedPersistenceVolumeClaimsFrom(pvcMap map[string]any, volumes []corev1.Volume) map[string]any { diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index adae4d6ae..dab75dfbf 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -999,11 +999,12 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { testEnv := getTestEnv() radixDeployment := buildRd(appName, environment, componentName, scenario.radixVolumeMounts) putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeclient, &scenario) - desiredDeployment := getDesiredDeployment(componentName, scenario.volumes) + desiredVolumes := getDesiredDeployment(componentName, scenario.volumes).Spec.Template.Spec.Volumes deployComponent := radixDeployment.Spec.Components[0] - err := CreateOrUpdateCsiAzureVolumeResources(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, environment, &deployComponent, desiredDeployment) + actualVolumes, err := CreateOrUpdateCsiAzureVolumeResources(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, environment, &deployComponent, desiredVolumes) assert.Nil(t, err) + assert.Equal(t, len(scenario.volumes), len(actualVolumes), "Number of volumes is not equal") existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) assert.Nil(t, err) From 7461bd8004c1beb0122ec8842151fdfff33b9cde Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 2 Jan 2025 17:02:41 +0100 Subject: [PATCH 39/68] Fixing not created job aux deployment --- pkg/apis/deployment/deployment.go | 11 +++++++---- pkg/apis/deployment/kubedeployment.go | 22 +++++++++------------- pkg/apis/deployment/kubedeployment_test.go | 5 +---- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 94a7b9c76..2cf417bab 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -589,7 +589,10 @@ func (deploy *Deployment) syncDeploymentForRadixComponent(ctx context.Context, c } func (deploy *Deployment) createOrUpdateJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, namespace, deploymentName string, volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) (*appsv1.Deployment, *appsv1.Deployment, error) { - currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.getCurrentAndDesiredJobAuxDeployment(ctx, deployComponent, namespace, deploymentName) + jobDeploymentName := deployComponent.GetName() + jobAuxDeploymentName := getJobAuxObjectName(jobDeploymentName) + + currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.getCurrentAndDesiredJobAuxDeployment(ctx, deployComponent, namespace, jobDeploymentName, jobAuxDeploymentName) if err != nil { return nil, nil, err } @@ -619,11 +622,11 @@ func syncRadixRestartEnvironmentVariable(deployComponent v1.RadixCommonDeployCom } } -func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, namespace, deploymentName string) (*appsv1.Deployment, *appsv1.Deployment, error) { - currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(ctx, namespace, deploymentName) +func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, namespace, jobDeploymentName, jobAuxDeploymentName string) (*appsv1.Deployment, *appsv1.Deployment, error) { + currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(ctx, namespace, jobAuxDeploymentName) if err != nil { if k8sErrors.IsNotFound(err) { - return nil, deploy.createJobAuxDeployment(deployComponent), nil + return nil, deploy.createJobAuxDeployment(jobDeploymentName, jobAuxDeploymentName), nil } return nil, nil, err } diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index 82dfd9ae8..f3d5644ca 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -25,7 +25,8 @@ import ( ) func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent) error { - currentDeployment, desiredDeployment, err := deploy.getCurrentAndDesiredDeployment(ctx, deployComponent) + namespace := deploy.radixDeployment.Namespace + currentDeployment, desiredDeployment, err := deploy.getCurrentAndDesiredDeployment(ctx, namespace, deployComponent) if err != nil { return err } @@ -45,17 +46,17 @@ func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployCompone } desiredDeployment.Spec.Template.Spec.Volumes = actualVolumes desiredVolumeMounts := desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts - if err = deploy.handleJobAuxDeployment(ctx, deployComponent, desiredDeployment, actualVolumes, desiredVolumeMounts); err != nil { + if err = deploy.handleJobAuxDeployment(ctx, namespace, deployComponent, desiredDeployment, actualVolumes, desiredVolumeMounts); err != nil { + //!!!! error "failed to create Deployment object: deployments.apps \"job1-aux\" already exists" return err } - return deploy.kubeutil.ApplyDeployment(ctx, deploy.radixDeployment.Namespace, currentDeployment, desiredDeployment) + return deploy.kubeutil.ApplyDeployment(ctx, namespace, currentDeployment, desiredDeployment) } -func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment, volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) error { +func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, namespace string, deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment, volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) error { if !internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { return nil } - namespace := desiredDeployment.GetNamespace() deploymentName := desiredDeployment.GetName() currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, namespace, deploymentName, volumes, volumeMounts) if err != nil { @@ -82,14 +83,11 @@ func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, deployComp return deploy.kubeutil.ApplyDeployment(ctx, deploy.radixDeployment.Namespace, currentJobAuxDeployment, desiredJobAuxDeployment) } -func (deploy *Deployment) getCurrentAndDesiredDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent) (*appsv1.Deployment, *appsv1.Deployment, error) { - namespace := deploy.radixDeployment.Namespace - +func (deploy *Deployment) getCurrentAndDesiredDeployment(ctx context.Context, namespace string, deployComponent v1.RadixCommonDeployComponent) (*appsv1.Deployment, *appsv1.Deployment, error) { currentDeployment, desiredDeployment, err := deploy.getDesiredDeployment(ctx, namespace, deployComponent) if err != nil { return nil, nil, err } - return currentDeployment, desiredDeployment, err } @@ -136,9 +134,7 @@ func (deploy *Deployment) getDesiredCreatedDeploymentConfig(ctx context.Context, return desiredDeployment, err } -func (deploy *Deployment) createJobAuxDeployment(deployComponent v1.RadixCommonDeployComponent) *appsv1.Deployment { - jobName := deployComponent.GetName() - jobAuxDeploymentName := getJobAuxObjectName(jobName) +func (deploy *Deployment) createJobAuxDeployment(jobName, jobAuxDeploymentName string) *appsv1.Deployment { desiredDeployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: jobAuxDeploymentName, @@ -504,7 +500,7 @@ func getContainerPorts(deployComponent v1.RadixCommonDeployComponent) []corev1.C for _, v := range componentPorts { containerPort := corev1.ContainerPort{ Name: v.Name, - ContainerPort: int32(v.Port), + ContainerPort: v.Port, Protocol: corev1.ProtocolTCP, } ports = append(ports, containerPort) diff --git a/pkg/apis/deployment/kubedeployment_test.go b/pkg/apis/deployment/kubedeployment_test.go index 54c694fa2..0b5f1fd2d 100644 --- a/pkg/apis/deployment/kubedeployment_test.go +++ b/pkg/apis/deployment/kubedeployment_test.go @@ -255,10 +255,7 @@ func applyDeploymentWithSyncWithComponentResources(t *testing.T, origRequests, o func TestDeployment_createJobAuxDeployment(t *testing.T) { deploy := &Deployment{radixDeployment: &v1.RadixDeployment{ObjectMeta: metav1.ObjectMeta{Name: "deployment1", UID: "uid1"}}} - jobDeployComponent := &v1.RadixDeployComponent{ - Name: "job1", - } - jobAuxDeployment := deploy.createJobAuxDeployment(jobDeployComponent) + jobAuxDeployment := deploy.createJobAuxDeployment("job1", "job1-aux") assert.Equal(t, "job1-aux", jobAuxDeployment.GetName()) resources := jobAuxDeployment.Spec.Template.Spec.Containers[0].Resources s := resources.Requests.Cpu().String() From 325c24f26f973a03f6615ca2324cdf027c9a56ec Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 3 Jan 2025 14:29:43 +0100 Subject: [PATCH 40/68] Fixed not created job volumes --- pkg/apis/batch/kubejob.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/apis/batch/kubejob.go b/pkg/apis/batch/kubejob.go index c0321ffc1..127f0b868 100644 --- a/pkg/apis/batch/kubejob.go +++ b/pkg/apis/batch/kubejob.go @@ -164,7 +164,6 @@ func (s *syncer) buildJob(ctx context.Context, batchJob *radixv1.RadixBatchJob, TTLSecondsAfterFinished: pointers.Ptr(int32(86400)), // delete completed job after 24 hours }, } - return job, nil } @@ -178,7 +177,7 @@ func (s *syncer) getJobPodImagePullSecrets(rd *radixv1.RadixDeployment) []corev1 func (s *syncer) appendPayloadSecretVolumes(batchJob *radixv1.RadixBatchJob, radixJobComponent *radixv1.RadixDeployJobComponent, volumes []corev1.Volume) []corev1.Volume { if radixJobComponent.Payload == nil || batchJob.PayloadSecretRef == nil { - return nil + return volumes } return append(volumes, corev1.Volume{ Name: jobPayloadVolumeName, From 6dec63189fa73370973437b0a4ee890dcc8d0869 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 3 Jan 2025 14:34:40 +0100 Subject: [PATCH 41/68] Extended job aux resource --- charts/radix-operator/Chart.yaml | 4 ++-- pkg/apis/deployment/kubedeployment.go | 2 +- pkg/apis/deployment/kubedeployment_test.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/radix-operator/Chart.yaml b/charts/radix-operator/Chart.yaml index eb740213c..f11a12a64 100644 --- a/charts/radix-operator/Chart.yaml +++ b/charts/radix-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: radix-operator -version: 1.48.2 -appVersion: 1.68.2 +version: 1.48.3 +appVersion: 1.68.3 kubeVersion: ">=1.24.0" description: Radix Operator keywords: diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index df5cb5df4..3b7e329ce 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -151,7 +151,7 @@ func (deploy *Deployment) createJobAuxDeployment(deployComponent v1.RadixCommonD Spec: corev1.PodSpec{Containers: []corev1.Container{ { Name: jobAuxDeploymentName, - Resources: resources.New(resources.WithCPUMilli(1), resources.WithMemoryMega(10)), + Resources: resources.New(resources.WithCPUMilli(1), resources.WithMemoryMega(20)), }}, }, }, diff --git a/pkg/apis/deployment/kubedeployment_test.go b/pkg/apis/deployment/kubedeployment_test.go index 54c694fa2..2d3d31cca 100644 --- a/pkg/apis/deployment/kubedeployment_test.go +++ b/pkg/apis/deployment/kubedeployment_test.go @@ -263,7 +263,7 @@ func TestDeployment_createJobAuxDeployment(t *testing.T) { resources := jobAuxDeployment.Spec.Template.Spec.Containers[0].Resources s := resources.Requests.Cpu().String() assert.Equal(t, "1m", s) - assert.Equal(t, "10M", resources.Requests.Memory().String()) + assert.Equal(t, "20M", resources.Requests.Memory().String()) assert.Equal(t, "0", resources.Limits.Cpu().String()) - assert.Equal(t, "10M", resources.Limits.Memory().String()) + assert.Equal(t, "20M", resources.Limits.Memory().String()) } From 8a9d402ffd48349dbd70f58084bf27c264121a72 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 3 Jan 2025 14:59:15 +0100 Subject: [PATCH 42/68] Extended job aux resource --- pkg/apis/deployment/deployment.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 8d40130da..e571532f7 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/equinor/radix-operator/pkg/apis/utils/resources" "sort" "strings" "time" @@ -639,7 +640,9 @@ func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Conte } return nil, nil, err } - return currentJobAuxDeployment, currentJobAuxDeployment.DeepCopy(), nil + desiredJobAuxDeployment := currentJobAuxDeployment.DeepCopy() + desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].Resources = resources.New(resources.WithCPUMilli(1), resources.WithMemoryMega(20)) + return currentJobAuxDeployment, desiredJobAuxDeployment, nil } func getJobAuxObjectName(jobName string) string { From f92d84be2a3b7b3a100b927429bce75a4c5ecdc0 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 3 Jan 2025 15:49:27 +0100 Subject: [PATCH 43/68] Cleanup --- pkg/apis/deployment/deployment.go | 20 +++++++++++--------- pkg/apis/deployment/kubedeployment.go | 6 +++--- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 2cf417bab..b8f4e1d86 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "sort" "strings" "time" @@ -15,10 +14,12 @@ import ( "github.com/equinor/radix-operator/pkg/apis/config" "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/ingress" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "github.com/equinor/radix-operator/pkg/apis/kube" "github.com/equinor/radix-operator/pkg/apis/metrics" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/utils/resources" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" monitoring "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" "github.com/rs/zerolog/log" @@ -588,11 +589,10 @@ func (deploy *Deployment) syncDeploymentForRadixComponent(ctx context.Context, c return nil } -func (deploy *Deployment) createOrUpdateJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, namespace, deploymentName string, volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) (*appsv1.Deployment, *appsv1.Deployment, error) { - jobDeploymentName := deployComponent.GetName() - jobAuxDeploymentName := getJobAuxObjectName(jobDeploymentName) +func (deploy *Deployment) createOrUpdateJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, namespace, jobKubeDeploymentName string, volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) (*appsv1.Deployment, *appsv1.Deployment, error) { + jobAuxKubeDeploymentName := getJobAuxObjectName(jobKubeDeploymentName) - currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.getCurrentAndDesiredJobAuxDeployment(ctx, deployComponent, namespace, jobDeploymentName, jobAuxDeploymentName) + currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.getCurrentAndDesiredJobAuxDeployment(ctx, namespace, jobKubeDeploymentName, jobAuxKubeDeploymentName) if err != nil { return nil, nil, err } @@ -622,15 +622,17 @@ func syncRadixRestartEnvironmentVariable(deployComponent v1.RadixCommonDeployCom } } -func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, namespace, jobDeploymentName, jobAuxDeploymentName string) (*appsv1.Deployment, *appsv1.Deployment, error) { - currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(ctx, namespace, jobAuxDeploymentName) +func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, namespace, jobKubeDeploymentName, jobAuxKubeDeploymentName string) (*appsv1.Deployment, *appsv1.Deployment, error) { + currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(ctx, namespace, jobAuxKubeDeploymentName) if err != nil { if k8sErrors.IsNotFound(err) { - return nil, deploy.createJobAuxDeployment(jobDeploymentName, jobAuxDeploymentName), nil + return nil, deploy.createJobAuxDeployment(jobKubeDeploymentName, jobAuxKubeDeploymentName), nil } return nil, nil, err } - return currentJobAuxDeployment, currentJobAuxDeployment.DeepCopy(), nil + desiredJobAuxDeployment := currentJobAuxDeployment.DeepCopy() + desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].Resources = resources.New(resources.WithCPUMilli(1), resources.WithMemoryMega(20)) + return currentJobAuxDeployment, desiredJobAuxDeployment, nil } func getJobAuxObjectName(jobName string) string { diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index da0d8eba0..6eda61759 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -57,8 +57,8 @@ func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, namespace if !internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { return nil } - deploymentName := desiredDeployment.GetName() - currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, namespace, deploymentName, volumes, volumeMounts) + jobKubeDeploymentName := desiredDeployment.GetName() + currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, namespace, jobKubeDeploymentName, volumes, volumeMounts) if err != nil { return err } @@ -71,7 +71,7 @@ func (deploy *Deployment) handleJobAuxDeployment(ctx context.Context, namespace return err } - currentJobAuxDeployment, desiredJobAuxDeployment, err = deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, namespace, deploymentName, volumes, volumeMounts) + currentJobAuxDeployment, desiredJobAuxDeployment, err = deploy.createOrUpdateJobAuxDeployment(ctx, deployComponent, namespace, jobKubeDeploymentName, volumes, volumeMounts) if err != nil { return err } From f8208d62bd564d5362c41e944fbdb7d69f4a9751 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 3 Jan 2025 15:57:52 +0100 Subject: [PATCH 44/68] Cleanup --- pkg/apis/deployment/deployment.go | 4 +--- pkg/apis/deployment/kubedeployment.go | 3 +-- pkg/apis/deployment/resources.go | 10 ++++++++++ 3 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 pkg/apis/deployment/resources.go diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 54a90e2a9..4d42bf30c 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/equinor/radix-operator/pkg/apis/utils/resources" "sort" "strings" "time" @@ -20,7 +19,6 @@ import ( "github.com/equinor/radix-operator/pkg/apis/metrics" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" - "github.com/equinor/radix-operator/pkg/apis/utils/resources" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" monitoring "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" "github.com/rs/zerolog/log" @@ -632,7 +630,7 @@ func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Conte return nil, nil, err } desiredJobAuxDeployment := currentJobAuxDeployment.DeepCopy() - desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].Resources = resources.New(resources.WithCPUMilli(1), resources.WithMemoryMega(20)) + desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].Resources = getJobAuxResources() return currentJobAuxDeployment, desiredJobAuxDeployment, nil } diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index 6eda61759..d3fde3e28 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -14,7 +14,6 @@ import ( "github.com/equinor/radix-operator/pkg/apis/utils" radixannotations "github.com/equinor/radix-operator/pkg/apis/utils/annotations" radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" - "github.com/equinor/radix-operator/pkg/apis/utils/resources" "github.com/rs/zerolog/log" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -150,7 +149,7 @@ func (deploy *Deployment) createJobAuxDeployment(jobName, jobAuxDeploymentName s Spec: corev1.PodSpec{Containers: []corev1.Container{ { Name: jobAuxDeploymentName, - Resources: resources.New(resources.WithCPUMilli(1), resources.WithMemoryMega(20)), + Resources: getJobAuxResources(), }}, }, }, diff --git a/pkg/apis/deployment/resources.go b/pkg/apis/deployment/resources.go new file mode 100644 index 000000000..5426e092c --- /dev/null +++ b/pkg/apis/deployment/resources.go @@ -0,0 +1,10 @@ +package deployment + +import ( + "github.com/equinor/radix-operator/pkg/apis/utils/resources" + "k8s.io/api/core/v1" +) + +func getJobAuxResources() v1.ResourceRequirements { + return resources.New(resources.WithCPUMilli(1), resources.WithMemoryMega(20)) +} From b649fe0a6522eee78a8662c86eb4618c4c641aba Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 3 Jan 2025 16:47:27 +0100 Subject: [PATCH 45/68] Reuse of job aux volumes --- pkg/apis/batch/syncer.go | 21 +++++++++++++++++++-- pkg/apis/defaults/jobscheduler.go | 7 +++++++ pkg/apis/deployment/deployment.go | 11 +++-------- pkg/apis/volumemount/volumemount.go | 9 +++++---- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/pkg/apis/batch/syncer.go b/pkg/apis/batch/syncer.go index 25a7b84d4..ec29408b4 100644 --- a/pkg/apis/batch/syncer.go +++ b/pkg/apis/batch/syncer.go @@ -6,12 +6,14 @@ import ( commonutils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/slice" "github.com/equinor/radix-operator/pkg/apis/config" + "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" "github.com/equinor/radix-operator/pkg/apis/volumemount" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" "github.com/rs/zerolog/log" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" @@ -96,8 +98,11 @@ func (s *syncer) reconcile(ctx context.Context) error { if err != nil { return err } - - desiredVolumes, err := volumemount.GetVolumes(ctx, s.kubeUtil, namespace, jobComponent, rd.Name, nil) + existingVolumes, err := getExistingVolumes(ctx, s.kubeUtil, namespace, jobComponent) + if err != nil { + return err + } + desiredVolumes, err := volumemount.GetVolumes(ctx, s.kubeUtil, namespace, jobComponent, rd.Name, existingVolumes) if err != nil { return err } @@ -125,6 +130,18 @@ func (s *syncer) reconcile(ctx context.Context) error { return nil } +func getExistingVolumes(ctx context.Context, kubeUtil *kube.Kube, namespace string, jobComponent *radixv1.RadixDeployJobComponent) ([]corev1.Volume, error) { + jobAuxKubeDeploymentName := defaults.GetJobAuxKubeDeployName(jobComponent.GetName()) + jobAuxKubeDeployment, err := kubeUtil.GetDeployment(ctx, namespace, jobAuxKubeDeploymentName) + if err != nil { + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return jobAuxKubeDeployment.Spec.Template.Spec.Volumes, nil +} + func (s *syncer) getRadixDeploymentAndJobComponent(ctx context.Context) (*radixv1.RadixDeployment, *radixv1.RadixDeployJobComponent, error) { rd, err := s.getRadixDeployment(ctx) if err != nil { diff --git a/pkg/apis/defaults/jobscheduler.go b/pkg/apis/defaults/jobscheduler.go index b9cd09f6f..dd4fb7da1 100644 --- a/pkg/apis/defaults/jobscheduler.go +++ b/pkg/apis/defaults/jobscheduler.go @@ -1,4 +1,11 @@ package defaults +import "fmt" + const RadixJobSchedulerPortName = "scheduler-port" const RadixJobTimeLimitSeconds = 43200 // 12 hours + +// GetJobAuxKubeDeployName Get the aux kube deployment name for a job component +func GetJobAuxKubeDeployName(jobName string) string { + return fmt.Sprintf("%s-aux", jobName) +} diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 4d42bf30c..250c0963f 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -589,9 +589,7 @@ func (deploy *Deployment) syncDeploymentForRadixComponent(ctx context.Context, c } func (deploy *Deployment) createOrUpdateJobAuxDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, namespace, jobKubeDeploymentName string, volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) (*appsv1.Deployment, *appsv1.Deployment, error) { - jobAuxKubeDeploymentName := getJobAuxObjectName(jobKubeDeploymentName) - - currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.getCurrentAndDesiredJobAuxDeployment(ctx, namespace, jobKubeDeploymentName, jobAuxKubeDeploymentName) + currentJobAuxDeployment, desiredJobAuxDeployment, err := deploy.getCurrentAndDesiredJobAuxDeployment(ctx, namespace, jobKubeDeploymentName) if err != nil { return nil, nil, err } @@ -621,7 +619,8 @@ func syncRadixRestartEnvironmentVariable(deployComponent v1.RadixCommonDeployCom } } -func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, namespace, jobKubeDeploymentName, jobAuxKubeDeploymentName string) (*appsv1.Deployment, *appsv1.Deployment, error) { +func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, namespace, jobKubeDeploymentName string) (*appsv1.Deployment, *appsv1.Deployment, error) { + jobAuxKubeDeploymentName := defaults.GetJobAuxKubeDeployName(jobKubeDeploymentName) currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(ctx, namespace, jobAuxKubeDeploymentName) if err != nil { if k8sErrors.IsNotFound(err) { @@ -633,7 +632,3 @@ func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Conte desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].Resources = getJobAuxResources() return currentJobAuxDeployment, desiredJobAuxDeployment, nil } - -func getJobAuxObjectName(jobName string) string { - return fmt.Sprintf("%s-aux", jobName) -} diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index c0e1e2c7b..d2d5ef1ea 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -166,7 +166,7 @@ func getRadixComponentVolumeMounts(deployComponent radixv1.RadixCommonDeployComp var volumeMounts []corev1.VolumeMount for _, volumeMount := range deployComponent.GetVolumeMounts() { - name, err := getVolumeMountVolumeName(&volumeMount, deployComponent.GetName()) + name, err := GetVolumeMountVolumeName(&volumeMount, deployComponent.GetName()) if err != nil { return nil, err } @@ -309,7 +309,7 @@ func getComponentVolumeMountVolumes(deployComponent radixv1.RadixCommonDeployCom } func createVolume(radixVolumeMount radixv1.RadixVolumeMount, componentName string, existingVolumeSourcesMap map[string]corev1.VolumeSource) (*corev1.Volume, error) { - volumeName, err := getVolumeMountVolumeName(&radixVolumeMount, componentName) + volumeName, err := GetVolumeMountVolumeName(&radixVolumeMount, componentName) if err != nil { return nil, err } @@ -378,7 +378,8 @@ func getComponentVolumeMountEmptyDirVolumeSource(spec *radixv1.RadixEmptyDirVolu } } -func getVolumeMountVolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { +// GetVolumeMountVolumeName Gets the volume name for a volume mount +func GetVolumeMountVolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { switch { case volumeMount.HasDeprecatedVolume(): return getVolumeMountDeprecatedVolumeName(volumeMount, componentName) @@ -853,7 +854,7 @@ func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeCli return nil, err } } - volume.PersistentVolumeClaim.ClaimName = pvcName + volume.PersistentVolumeClaim.ClaimName = pvcName // in case it was updated with new name return &volume, nil } From 1ce04a331e14b28153cd5c227eed4fd6a2713847 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 6 Jan 2025 14:02:50 +0100 Subject: [PATCH 46/68] Fixed unit-tests --- pkg/apis/volumemount/volumemount_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index dab75dfbf..7c1ebbc2d 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -1002,7 +1002,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { desiredVolumes := getDesiredDeployment(componentName, scenario.volumes).Spec.Template.Spec.Volumes deployComponent := radixDeployment.Spec.Components[0] - actualVolumes, err := CreateOrUpdateCsiAzureVolumeResources(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, environment, &deployComponent, desiredVolumes) + actualVolumes, err := CreateOrUpdateCsiAzureVolumeResources(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, utils.GetEnvironmentNamespace(appName, environment), &deployComponent, desiredVolumes) assert.Nil(t, err) assert.Equal(t, len(scenario.volumes), len(actualVolumes), "Number of volumes is not equal") @@ -1073,7 +1073,7 @@ func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcPvProperties)) expec props := expectedPvcPvProperties{ appName: appName, environment: environment, - namespace: fmt.Sprintf("%s-%s", appName, environment), + namespace: utils.GetEnvironmentNamespace(appName, environment), componentName: componentName, radixVolumeMountName: "volume1", radixStorageName: "storage1", @@ -1404,6 +1404,7 @@ func createExpectedPvc(props expectedPvcPvProperties, modify func(*corev1.Persis Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse(props.requestsVolumeMountSize)}, // it seems correct number is not needed for CSI driver }, VolumeName: props.persistentVolumeName, + VolumeMode: pointers.Ptr(corev1.PersistentVolumeFilesystem), }, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } @@ -1538,11 +1539,15 @@ func equalPersistentVolumeClaims(pvcList1, pvcList2 *[]corev1.PersistentVolumeCl } func equalVolumeNamePrefix(pvc1 corev1.PersistentVolumeClaim, pvc2 corev1.PersistentVolumeClaim) bool { - return pvc1.Spec.VolumeName[:20] == pvc2.Spec.VolumeName[:20] + prefix1 := pvc1.Spec.VolumeName[:20] + prefix2 := pvc2.Spec.VolumeName[:20] + return prefix1 == prefix2 } func equalNamePrefix(pvc1 corev1.PersistentVolumeClaim, pvc2 corev1.PersistentVolumeClaim) bool { - return pvc1.GetName()[:len(pvc1.GetName())-5] == pvc2.GetName()[:len(pvc2.GetName())-5] + prefix1 := pvc1.GetName()[:len(pvc1.GetName())-5] + prefix2 := pvc2.GetName()[:len(pvc2.GetName())-5] + return prefix1 == prefix2 } func getPvcCopyWithLabels(pvc *corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (*corev1.PersistentVolumeClaim, map[string]string) { From 47a5caacbe39825151767a7f7a37b010808c80c7 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 6 Jan 2025 14:34:18 +0100 Subject: [PATCH 47/68] Fixing unit-tests --- pkg/apis/deployment/azurekeyvault_test.go | 157 +++++++++++----------- pkg/apis/volumemount/volumemount_test.go | 2 +- 2 files changed, 81 insertions(+), 78 deletions(-) diff --git a/pkg/apis/deployment/azurekeyvault_test.go b/pkg/apis/deployment/azurekeyvault_test.go index dfe052e9f..f1e9a814f 100644 --- a/pkg/apis/deployment/azurekeyvault_test.go +++ b/pkg/apis/deployment/azurekeyvault_test.go @@ -2,6 +2,7 @@ package deployment import ( "context" + "github.com/stretchr/testify/require" "strings" "testing" @@ -15,7 +16,8 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { appName := "app" namespace := "some-namespace" environment := "some-env" - componentName1, componentNameLong := "component1", "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit" + componentName1 := "component1" + //componentNameLong := "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit" type expectedVolumeProps struct { expectedVolumeNamePrefix string expectedVolumeMountPath string @@ -30,90 +32,91 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { expectedVolumeProps []expectedVolumeProps radixVolumeMounts []v1.RadixVolumeMount }{ - { - name: "No Azure Key volumes as no RadixAzureKeyVault-s", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{}, - expectedVolumeProps: []expectedVolumeProps{}, - }, + //{ + // name: "No Azure Key volumes as no RadixAzureKeyVault-s", + // componentName: componentName1, + // azureKeyVaults: []v1.RadixAzureKeyVault{}, + // expectedVolumeProps: []expectedVolumeProps{}, + //}, { name: "No Azure Key volumes as no secret names in secret object", componentName: componentName1, azureKeyVaults: []v1.RadixAzureKeyVault{{Name: "kv1"}}, }, - { - name: "One Azure Key volume for one secret objects secret name", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{{ - Name: "kv1", - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - }}, - expectedVolumeProps: []expectedVolumeProps{ - { - expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", - expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", - expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "component1-az-keyvault-kv1-", - }, - }, - }, - }, - { - name: "Multiple Azure Key volumes for each RadixAzureKeyVault", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{ - { - Name: "kv1", - Path: utils.StringPtr("/mnt/customPath"), - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - }, - { - Name: "kv2", - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret2", EnvVar: "SECRET_REF2"}}, - }, - }, - expectedVolumeProps: []expectedVolumeProps{ - { - expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", - expectedVolumeMountPath: "/mnt/customPath", - expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "component1-az-keyvault-kv1-", - }, - }, - { - expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv2-", - expectedVolumeMountPath: "/mnt/azure-key-vault/kv2", - expectedNodePublishSecretRefName: "component1-kv2-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "component1-az-keyvault-kv2-", - }, - }, - }, - }, - { - name: "Volume name should be trimmed when exceeding 63 chars", - componentName: componentNameLong, - azureKeyVaults: []v1.RadixAzureKeyVault{{ - Name: "kv1", - Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - }}, - expectedVolumeProps: []expectedVolumeProps{ - { - expectedVolumeNamePrefix: "a-very-long-component-name-that-exceeds-63-kubernetes-vol", - expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", - expectedNodePublishSecretRefName: "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-kv1-csiazkvcreds", - expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-az-keyvault-kv1-", - }, - }, - }, - }, + //{ + // name: "One Azure Key volume for one secret objects secret name", + // componentName: componentName1, + // azureKeyVaults: []v1.RadixAzureKeyVault{{ + // Name: "kv1", + // Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + // }}, + // expectedVolumeProps: []expectedVolumeProps{ + // { + // expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", + // expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", + // expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", + // expectedVolumeAttributePrefixes: map[string]string{ + // "secretProviderClass": "component1-az-keyvault-kv1-", + // }, + // }, + // }, + //}, + //{ + // name: "Multiple Azure Key volumes for each RadixAzureKeyVault", + // componentName: componentName1, + // azureKeyVaults: []v1.RadixAzureKeyVault{ + // { + // Name: "kv1", + // Path: utils.StringPtr("/mnt/customPath"), + // Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + // }, + // { + // Name: "kv2", + // Items: []v1.RadixAzureKeyVaultItem{{Name: "secret2", EnvVar: "SECRET_REF2"}}, + // }, + // }, + // expectedVolumeProps: []expectedVolumeProps{ + // { + // expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", + // expectedVolumeMountPath: "/mnt/customPath", + // expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", + // expectedVolumeAttributePrefixes: map[string]string{ + // "secretProviderClass": "component1-az-keyvault-kv1-", + // }, + // }, + // { + // expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv2-", + // expectedVolumeMountPath: "/mnt/azure-key-vault/kv2", + // expectedNodePublishSecretRefName: "component1-kv2-csiazkvcreds", + // expectedVolumeAttributePrefixes: map[string]string{ + // "secretProviderClass": "component1-az-keyvault-kv2-", + // }, + // }, + // }, + //}, + //{ + // name: "Volume name should be trimmed when exceeding 63 chars", + // componentName: componentNameLong, + // azureKeyVaults: []v1.RadixAzureKeyVault{{ + // Name: "kv1", + // Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + // }}, + // expectedVolumeProps: []expectedVolumeProps{ + // { + // expectedVolumeNamePrefix: "a-very-long-component-name-that-exceeds-63-kubernetes-vol", + // expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", + // expectedNodePublishSecretRefName: "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-kv1-csiazkvcreds", + // expectedVolumeAttributePrefixes: map[string]string{ + // "secretProviderClass": "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-az-keyvault-kv1-", + // }, + // }, + // }, + //}, } t.Run("CSI Azure Key vault volumes", func(t *testing.T) { t.Parallel() for _, scenario := range scenarios { + t.Logf("Test case %s", scenario.name) deployment := getDeployment(t) radixDeployment := buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { var builders []utils.DeployComponentBuilder @@ -132,7 +135,7 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { } } volumes, err := volumemount.GetVolumes(context.Background(), deployment.kubeutil, namespace, radixDeployComponent, radixDeployment.GetName(), nil) - assert.Nil(t, err) + require.NoError(t, err, "failed to get volumes") assert.Len(t, volumes, len(scenario.expectedVolumeProps)) if len(scenario.expectedVolumeProps) == 0 { continue diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index 7c1ebbc2d..9d33cbe8e 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -993,7 +993,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }()...) suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { - for _, factory := range suite.radixCommonDeployComponentFactories[:1] { + for _, factory := range suite.radixCommonDeployComponentFactories { for _, scenario := range scenarios { t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) testEnv := getTestEnv() From 75167949191f3d48097bb9006dd47f226c9d98a8 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 6 Jan 2025 16:01:17 +0100 Subject: [PATCH 48/68] Fixing unit-tests --- pkg/apis/deployment/azurekeyvault_test.go | 8 ++++---- pkg/apis/kube/secret_provider.go | 3 +-- pkg/apis/utils/branch/path_matcher_test.go | 11 +++++++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/apis/deployment/azurekeyvault_test.go b/pkg/apis/deployment/azurekeyvault_test.go index f1e9a814f..e85be96c6 100644 --- a/pkg/apis/deployment/azurekeyvault_test.go +++ b/pkg/apis/deployment/azurekeyvault_test.go @@ -117,7 +117,7 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { t.Parallel() for _, scenario := range scenarios { t.Logf("Test case %s", scenario.name) - deployment := getDeployment(t) + deployment := getDeployment(t, environment) radixDeployment := buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { var builders []utils.DeployComponentBuilder builders = append(builders, utils.NewDeployComponentBuilder(). @@ -164,7 +164,7 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { t.Run("CSI Azure Key vault volume mounts", func(t *testing.T) { t.Parallel() for _, scenario := range scenarios { - deployment := getDeployment(t) + deployment := getDeployment(t, environment) radixDeployment := buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { var builders []utils.DeployComponentBuilder builders = append(builders, utils.NewDeployComponentBuilder(). @@ -200,14 +200,14 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { }) } -func getDeployment(t *testing.T) *Deployment { +func getDeployment(t *testing.T, environment string) *Deployment { tu, client, kubeUtil, radixClient, kedaClient, prometheusClient, _, certClient := SetupTest(t) rd, _ := ApplyDeploymentWithSync(tu, client, kubeUtil, radixClient, kedaClient, prometheusClient, certClient, utils.ARadixDeployment(). WithComponents(utils.NewDeployComponentBuilder(). WithName("comp1")). WithAppName("any-app"). - WithEnvironment("test")) + WithEnvironment(environment)) return &Deployment{radixclient: radixClient, kubeutil: kubeUtil, radixDeployment: rd, config: &testConfig} } diff --git a/pkg/apis/kube/secret_provider.go b/pkg/apis/kube/secret_provider.go index aa2e8a222..20307f293 100644 --- a/pkg/apis/kube/secret_provider.go +++ b/pkg/apis/kube/secret_provider.go @@ -61,8 +61,7 @@ func GetComponentSecretProviderClassName(radixDeploymentName, radixDeployCompone // by naming component the same as secret-ref object hash := strings.ToLower(commonUtils.RandStringStrSeed(5, strings.ToLower(fmt.Sprintf("%s-%s-%s-%s", radixDeployComponentName, radixDeploymentName, radixSecretRefType, secretRefName)))) - return strings.ToLower(fmt.Sprintf("%s-%s-%s-%s", radixDeployComponentName, radixSecretRefType, secretRefName, - hash)) + return strings.ToLower(fmt.Sprintf("%s-%s-%s-%s", radixDeployComponentName, radixSecretRefType, secretRefName, hash)) } // BuildAzureKeyVaultSecretProviderClass Build a SecretProviderClass for Azure Key vault secret-ref diff --git a/pkg/apis/utils/branch/path_matcher_test.go b/pkg/apis/utils/branch/path_matcher_test.go index 0c231f46a..c2c78cbda 100644 --- a/pkg/apis/utils/branch/path_matcher_test.go +++ b/pkg/apis/utils/branch/path_matcher_test.go @@ -33,6 +33,11 @@ func TestMatchesPattern(t *testing.T) { assert.False(t, MatchesPattern("release/*", "release")) assert.False(t, MatchesPattern("release/**/*", "release")) assert.False(t, MatchesPattern("test/*/tull", "test/test1/test2/tull")) + assert.False(t, MatchesPattern("(test)|(main)/*", "release/t")) + assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1q.0.2")) + assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1..2")) + assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1.2")) + assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+\\.*", "v1.2.20-asdf")) assert.True(t, MatchesPattern("test", "test")) assert.True(t, MatchesPattern("te??", "test")) @@ -48,8 +53,6 @@ func TestMatchesPattern(t *testing.T) { assert.True(t, MatchesPattern("release/**/*", "release/q3/0.1.3")) assert.True(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1.0.2")) assert.True(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v123.033.2112")) - assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1q.0.2")) - assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1..2")) - assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+", "v1.2")) - assert.False(t, MatchesPattern("v\\d+\\.\\d+\\.\\d+\\.*", "v1.2.20-asdf")) + assert.True(t, MatchesPattern("(test)|(main)/*", "test/t")) + assert.True(t, MatchesPattern("(test)|(main)/*", "main/t")) } From 10e36f379e95eccc6d978073c6c153ad4487eb28 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 6 Jan 2025 17:00:25 +0100 Subject: [PATCH 49/68] Fixing unit-tests --- pkg/apis/deployment/azurekeyvault_test.go | 187 +++++++++++----------- pkg/apis/test/utils.go | 6 +- 2 files changed, 100 insertions(+), 93 deletions(-) diff --git a/pkg/apis/deployment/azurekeyvault_test.go b/pkg/apis/deployment/azurekeyvault_test.go index e85be96c6..2e29c4ad3 100644 --- a/pkg/apis/deployment/azurekeyvault_test.go +++ b/pkg/apis/deployment/azurekeyvault_test.go @@ -13,11 +13,13 @@ import ( ) func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { - appName := "app" - namespace := "some-namespace" - environment := "some-env" - componentName1 := "component1" - //componentNameLong := "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit" + const ( + appName = "app" + namespace = "some-namespace" + environment = "some-env" + componentName1 = "component1" + componentNameLong = "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit" + ) type expectedVolumeProps struct { expectedVolumeNamePrefix string expectedVolumeMountPath string @@ -38,11 +40,11 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { // azureKeyVaults: []v1.RadixAzureKeyVault{}, // expectedVolumeProps: []expectedVolumeProps{}, //}, - { - name: "No Azure Key volumes as no secret names in secret object", - componentName: componentName1, - azureKeyVaults: []v1.RadixAzureKeyVault{{Name: "kv1"}}, - }, + //{ + // name: "No Azure Key volumes as no secret names in secret object", + // componentName: componentName1, + // azureKeyVaults: []v1.RadixAzureKeyVault{{Name: "kv1"}}, + //}, //{ // name: "One Azure Key volume for one secret objects secret name", // componentName: componentName1, @@ -94,85 +96,85 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { // }, // }, //}, - //{ - // name: "Volume name should be trimmed when exceeding 63 chars", - // componentName: componentNameLong, - // azureKeyVaults: []v1.RadixAzureKeyVault{{ - // Name: "kv1", - // Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - // }}, - // expectedVolumeProps: []expectedVolumeProps{ - // { - // expectedVolumeNamePrefix: "a-very-long-component-name-that-exceeds-63-kubernetes-vol", - // expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", - // expectedNodePublishSecretRefName: "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-kv1-csiazkvcreds", - // expectedVolumeAttributePrefixes: map[string]string{ - // "secretProviderClass": "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-az-keyvault-kv1-", - // }, - // }, - // }, - //}, + { + name: "Volume name should be trimmed when exceeding 63 chars", + componentName: componentNameLong, + azureKeyVaults: []v1.RadixAzureKeyVault{{ + Name: "kv1", + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + }}, + expectedVolumeProps: []expectedVolumeProps{ + { + expectedVolumeNamePrefix: "a-very-long-component-name-that-exceeds-63-kubernetes-vol", + expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", + expectedNodePublishSecretRefName: "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-kv1-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-az-keyvault-kv1-", + }, + }, + }, + }, } - t.Run("CSI Azure Key vault volumes", func(t *testing.T) { - t.Parallel() - for _, scenario := range scenarios { - t.Logf("Test case %s", scenario.name) - deployment := getDeployment(t, environment) - radixDeployment := buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { - var builders []utils.DeployComponentBuilder - builders = append(builders, utils.NewDeployComponentBuilder(). - WithName(scenario.componentName). - WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) - return builders - }) - radixDeployComponent := radixDeployment.GetComponentByName(scenario.componentName) - for _, azureKeyVault := range scenario.azureKeyVaults { - spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) - if err != nil { - t.Log(err.Error()) - } else { - t.Logf("created secret provider class %s", spc.Name) - } - } - volumes, err := volumemount.GetVolumes(context.Background(), deployment.kubeutil, namespace, radixDeployComponent, radixDeployment.GetName(), nil) - require.NoError(t, err, "failed to get volumes") - assert.Len(t, volumes, len(scenario.expectedVolumeProps)) - if len(scenario.expectedVolumeProps) == 0 { - continue - } - - for i := 0; i < len(volumes); i++ { - volume := volumes[i] - assert.Less(t, len(volume.Name), 64, "volume name is too long") - assert.NotNil(t, volume.CSI) - assert.NotNil(t, volume.CSI.VolumeAttributes) - assert.NotNil(t, volume.CSI.NodePublishSecretRef) - assert.Equal(t, "secrets-store.csi.k8s.io", volume.CSI.Driver) - - volumeProp := scenario.expectedVolumeProps[i] - for attrKey, attrValue := range volumeProp.expectedVolumeAttributePrefixes { - spcValue, exists := volume.CSI.VolumeAttributes[attrKey] - assert.True(t, exists) - assert.True(t, strings.HasPrefix(spcValue, attrValue)) - } - assert.True(t, strings.Contains(volume.Name, volumeProp.expectedVolumeNamePrefix)) - assert.Equal(t, volumeProp.expectedNodePublishSecretRefName, volume.CSI.NodePublishSecretRef.Name) - } - } - }) + //t.Run("CSI Azure Key vault volumes", func(t *testing.T) { + // t.Parallel() + // for _, scenario := range scenarios { + // t.Logf("Test case %s", scenario.name) + // rdBuilder := getRdBuilderWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { + // var builders []utils.DeployComponentBuilder + // builders = append(builders, utils.NewDeployComponentBuilder(). + // WithName(trimComponentName(scenario.componentName)). + // WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) + // return builders + // }) + // deployment := getDeployment(t, rdBuilder) + // radixDeployComponent := deployment.radixDeployment.GetComponentByName(trimComponentName(scenario.componentName)) + // for _, azureKeyVault := range scenario.azureKeyVaults { + // spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) + // if err != nil { + // t.Log(err.Error()) + // } else { + // t.Logf("created secret provider class %s", spc.Name) + // } + // } + // volumes, err := volumemount.GetVolumes(context.Background(), deployment.kubeutil, namespace, radixDeployComponent, deployment.radixDeployment.GetName(), nil) + // require.NoError(t, err, "failed to get volumes") + // assert.Len(t, volumes, len(scenario.expectedVolumeProps)) + // if len(scenario.expectedVolumeProps) == 0 { + // continue + // } + // + // for i := 0; i < len(volumes); i++ { + // volume := volumes[i] + // assert.Less(t, len(volume.Name), 64, "volume name is too long") + // assert.NotNil(t, volume.CSI) + // assert.NotNil(t, volume.CSI.VolumeAttributes) + // assert.NotNil(t, volume.CSI.NodePublishSecretRef) + // assert.Equal(t, "secrets-store.csi.k8s.io", volume.CSI.Driver) + // + // volumeProp := scenario.expectedVolumeProps[i] + // for attrKey, attrValue := range volumeProp.expectedVolumeAttributePrefixes { + // spcValue, exists := volume.CSI.VolumeAttributes[attrKey] + // assert.True(t, exists) + // assert.True(t, strings.HasPrefix(spcValue, attrValue)) + // } + // assert.True(t, strings.Contains(volume.Name, volumeProp.expectedVolumeNamePrefix)) + // assert.Equal(t, volumeProp.expectedNodePublishSecretRefName, volume.CSI.NodePublishSecretRef.Name) + // } + // } + //}) t.Run("CSI Azure Key vault volume mounts", func(t *testing.T) { - t.Parallel() + //t.Parallel() for _, scenario := range scenarios { - deployment := getDeployment(t, environment) - radixDeployment := buildRdWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { + rdBuilder := getRdBuilderWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { var builders []utils.DeployComponentBuilder builders = append(builders, utils.NewDeployComponentBuilder(). - WithName(scenario.componentName). + WithName(trimComponentName(scenario.componentName)). WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) return builders }) - radixDeployComponent := radixDeployment.GetComponentByName(scenario.componentName) + deployment := getDeployment(t, rdBuilder) + radixDeployComponent := deployment.radixDeployment.GetComponentByName(trimComponentName(scenario.componentName)) for _, azureKeyVault := range scenario.azureKeyVaults { spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) if err != nil { @@ -181,7 +183,7 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { t.Logf("created secret provider class %s", spc.Name) } } - volumeMounts, err := volumemount.GetRadixDeployComponentVolumeMounts(radixDeployComponent, radixDeployment.GetName()) + volumeMounts, err := volumemount.GetRadixDeployComponentVolumeMounts(radixDeployComponent, deployment.radixDeployment.GetName()) assert.Nil(t, err) assert.Len(t, volumeMounts, len(scenario.expectedVolumeProps)) if len(scenario.expectedVolumeProps) == 0 { @@ -200,21 +202,24 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { }) } -func getDeployment(t *testing.T, environment string) *Deployment { +func trimComponentName(componentName string) string { + if len(componentName) > 50 { + return componentName[:50] + } + return componentName +} + +func getDeployment(t *testing.T, deploymentBuilder utils.DeploymentBuilder) *Deployment { tu, client, kubeUtil, radixClient, kedaClient, prometheusClient, _, certClient := SetupTest(t) - rd, _ := ApplyDeploymentWithSync(tu, client, kubeUtil, radixClient, kedaClient, prometheusClient, certClient, - utils.ARadixDeployment(). - WithComponents(utils.NewDeployComponentBuilder(). - WithName("comp1")). - WithAppName("any-app"). - WithEnvironment(environment)) + rd, err := ApplyDeploymentWithSync(tu, client, kubeUtil, radixClient, kedaClient, prometheusClient, certClient, + deploymentBuilder) + require.NoError(t, err) return &Deployment{radixclient: radixClient, kubeutil: kubeUtil, radixDeployment: rd, config: &testConfig} } -func buildRdWithComponentBuilders(appName string, environment string, componentBuilders func() []utils.DeployComponentBuilder) *v1.RadixDeployment { +func getRdBuilderWithComponentBuilders(appName string, environment string, componentBuilders func() []utils.DeployComponentBuilder) utils.DeploymentBuilder { return utils.ARadixDeployment(). WithAppName(appName). WithEnvironment(environment). - WithComponents(componentBuilders()...). - BuildRD() + WithComponents(componentBuilders()...) } diff --git a/pkg/apis/test/utils.go b/pkg/apis/test/utils.go index be897bf10..703ef2898 100644 --- a/pkg/apis/test/utils.go +++ b/pkg/apis/test/utils.go @@ -3,6 +3,7 @@ package test import ( "context" "errors" + commonUtils "github.com/equinor/radix-common/utils" "os" "time" @@ -170,8 +171,9 @@ func (tu *Utils) ApplyApplicationUpdate(applicationBuilder utils.ApplicationBuil // ApplyDeployment Will help persist a deployment func (tu *Utils) ApplyDeployment(ctx context.Context, deploymentBuilder utils.DeploymentBuilder) (*radixv1.RadixDeployment, error) { envs := make(map[string]struct{}) - if deploymentBuilder.GetApplicationBuilder() != nil { - ra, _ := tu.ApplyApplication(deploymentBuilder.GetApplicationBuilder()) + applicationBuilder := deploymentBuilder.GetApplicationBuilder() + if !commonUtils.IsNil(applicationBuilder) { + ra, _ := tu.ApplyApplication(applicationBuilder) for _, env := range ra.Spec.Environments { envs[env.Name] = struct{}{} } From a33a8653b5f1b8a502773751a7e0d62d5d173e3f Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 7 Jan 2025 09:24:31 +0100 Subject: [PATCH 50/68] Fixing unit-tests --- pkg/apis/deployment/azurekeyvault_test.go | 31 ++++++++++------------- pkg/apis/volumemount/volumemount.go | 7 ++--- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/pkg/apis/deployment/azurekeyvault_test.go b/pkg/apis/deployment/azurekeyvault_test.go index 2e29c4ad3..48e94317a 100644 --- a/pkg/apis/deployment/azurekeyvault_test.go +++ b/pkg/apis/deployment/azurekeyvault_test.go @@ -17,8 +17,7 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { appName = "app" namespace = "some-namespace" environment = "some-env" - componentName1 = "component1" - componentNameLong = "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit" + componentNameLong = "max-long-component-name-0123456789012345678901234" ) type expectedVolumeProps struct { expectedVolumeNamePrefix string @@ -105,11 +104,11 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { }}, expectedVolumeProps: []expectedVolumeProps{ { - expectedVolumeNamePrefix: "a-very-long-component-name-that-exceeds-63-kubernetes-vol", + expectedVolumeNamePrefix: "max-long-component-name-0123456789012345678901234-az-keyv", expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", - expectedNodePublishSecretRefName: "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-kv1-csiazkvcreds", + expectedNodePublishSecretRefName: "max-long-component-name-0123456789012345678901234-kv1-csiazkvcreds", expectedVolumeAttributePrefixes: map[string]string{ - "secretProviderClass": "a-very-long-component-name-that-exceeds-63-kubernetes-volume-name-limit-az-keyvault-kv1-", + "secretProviderClass": "max-long-component-name-0123456789012345678901234-az-keyvault-kv1-", }, }, }, @@ -169,12 +168,12 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { rdBuilder := getRdBuilderWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { var builders []utils.DeployComponentBuilder builders = append(builders, utils.NewDeployComponentBuilder(). - WithName(trimComponentName(scenario.componentName)). + WithName(scenario.componentName). WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) return builders }) deployment := getDeployment(t, rdBuilder) - radixDeployComponent := deployment.radixDeployment.GetComponentByName(trimComponentName(scenario.componentName)) + radixDeployComponent := deployment.radixDeployment.GetComponentByName(scenario.componentName) for _, azureKeyVault := range scenario.azureKeyVaults { spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) if err != nil { @@ -183,8 +182,10 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { t.Logf("created secret provider class %s", spc.Name) } } + volumeMounts, err := volumemount.GetRadixDeployComponentVolumeMounts(radixDeployComponent, deployment.radixDeployment.GetName()) - assert.Nil(t, err) + require.Nil(t, err) + assert.Len(t, volumeMounts, len(scenario.expectedVolumeProps)) if len(scenario.expectedVolumeProps) == 0 { continue @@ -194,21 +195,15 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { volumeMount := volumeMounts[i] volumeProp := scenario.expectedVolumeProps[i] assert.Less(t, len(volumeMount.Name), 64, "volumemount name is too long") - assert.True(t, strings.Contains(volumeMount.Name, volumeProp.expectedVolumeNamePrefix)) - assert.Equal(t, volumeProp.expectedVolumeMountPath, volumeMount.MountPath) - assert.True(t, volumeMount.ReadOnly) + assert.True(t, strings.Contains(volumeMount.Name, volumeProp.expectedVolumeNamePrefix), + "VolumeMount name should have prefix %s, but it is %s", volumeProp.expectedVolumeNamePrefix, volumeMount.Name) + assert.Equal(t, volumeProp.expectedVolumeMountPath, volumeMount.MountPath, "VolumeMount mount path should be %s, but it is %s", volumeProp.expectedVolumeMountPath, volumeMount.MountPath) + assert.True(t, volumeMount.ReadOnly, "VolumeMount should be read only") } } }) } -func trimComponentName(componentName string) string { - if len(componentName) > 50 { - return componentName[:50] - } - return componentName -} - func getDeployment(t *testing.T, deploymentBuilder utils.DeploymentBuilder) *Deployment { tu, client, kubeUtil, radixClient, kedaClient, prometheusClient, _, certClient := SetupTest(t) rd, err := ApplyDeploymentWithSync(tu, client, kubeUtil, radixClient, kedaClient, prometheusClient, certClient, diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index d2d5ef1ea..e387cd0b7 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -239,8 +239,8 @@ func getCsiRadixVolumeTypeId(radixVolumeMount *radixv1.RadixVolumeMount) (string return "", fmt.Errorf("unknown volume mount type %s", radixVolumeMount.Type) } -func getComponentSecretRefsVolumes(ctx context.Context, kubeutil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.Volume, error) { - azureKeyVaultVolumes, err := getComponentSecretRefsAzureKeyVaultVolumes(ctx, kubeutil, namespace, deployComponent, radixDeploymentName) +func getComponentSecretRefsVolumes(ctx context.Context, kubeUtil *kube.Kube, namespace string, deployComponent radixv1.RadixCommonDeployComponent, radixDeploymentName string) ([]corev1.Volume, error) { + azureKeyVaultVolumes, err := getComponentSecretRefsAzureKeyVaultVolumes(ctx, kubeUtil, namespace, deployComponent, radixDeploymentName) if err != nil { return nil, err } @@ -982,7 +982,8 @@ func trimVolumeNameToValidLength(volumeName string) string { } randString := strings.ToLower(commonUtils.RandStringStrSeed(randSize, volumeName)) - return fmt.Sprintf("%s-%s", volumeName[:63-randSize-1], randString) + sprintf := fmt.Sprintf("%s-%s", volumeName[:63-randSize-1], randString) + return sprintf } func getCsiAzureVolumeMountCredsSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace, componentName, volumeMountName string) (string, []byte, []byte) { From 0ec64253209cbeaa6cbdeb83f9bc8e2ba0c5ca56 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 7 Jan 2025 09:35:32 +0100 Subject: [PATCH 51/68] Fixing unit-tests --- pkg/apis/deployment/azurekeyvault_test.go | 222 +++++++++++----------- 1 file changed, 112 insertions(+), 110 deletions(-) diff --git a/pkg/apis/deployment/azurekeyvault_test.go b/pkg/apis/deployment/azurekeyvault_test.go index 48e94317a..32c12df41 100644 --- a/pkg/apis/deployment/azurekeyvault_test.go +++ b/pkg/apis/deployment/azurekeyvault_test.go @@ -2,6 +2,7 @@ package deployment import ( "context" + "github.com/equinor/radix-operator/pkg/apis/internal/persistentvolume" "github.com/stretchr/testify/require" "strings" "testing" @@ -17,6 +18,7 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { appName = "app" namespace = "some-namespace" environment = "some-env" + componentName1 = "component1" componentNameLong = "max-long-component-name-0123456789012345678901234" ) type expectedVolumeProps struct { @@ -33,68 +35,68 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { expectedVolumeProps []expectedVolumeProps radixVolumeMounts []v1.RadixVolumeMount }{ - //{ - // name: "No Azure Key volumes as no RadixAzureKeyVault-s", - // componentName: componentName1, - // azureKeyVaults: []v1.RadixAzureKeyVault{}, - // expectedVolumeProps: []expectedVolumeProps{}, - //}, - //{ - // name: "No Azure Key volumes as no secret names in secret object", - // componentName: componentName1, - // azureKeyVaults: []v1.RadixAzureKeyVault{{Name: "kv1"}}, - //}, - //{ - // name: "One Azure Key volume for one secret objects secret name", - // componentName: componentName1, - // azureKeyVaults: []v1.RadixAzureKeyVault{{ - // Name: "kv1", - // Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - // }}, - // expectedVolumeProps: []expectedVolumeProps{ - // { - // expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", - // expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", - // expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", - // expectedVolumeAttributePrefixes: map[string]string{ - // "secretProviderClass": "component1-az-keyvault-kv1-", - // }, - // }, - // }, - //}, - //{ - // name: "Multiple Azure Key volumes for each RadixAzureKeyVault", - // componentName: componentName1, - // azureKeyVaults: []v1.RadixAzureKeyVault{ - // { - // Name: "kv1", - // Path: utils.StringPtr("/mnt/customPath"), - // Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, - // }, - // { - // Name: "kv2", - // Items: []v1.RadixAzureKeyVaultItem{{Name: "secret2", EnvVar: "SECRET_REF2"}}, - // }, - // }, - // expectedVolumeProps: []expectedVolumeProps{ - // { - // expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", - // expectedVolumeMountPath: "/mnt/customPath", - // expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", - // expectedVolumeAttributePrefixes: map[string]string{ - // "secretProviderClass": "component1-az-keyvault-kv1-", - // }, - // }, - // { - // expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv2-", - // expectedVolumeMountPath: "/mnt/azure-key-vault/kv2", - // expectedNodePublishSecretRefName: "component1-kv2-csiazkvcreds", - // expectedVolumeAttributePrefixes: map[string]string{ - // "secretProviderClass": "component1-az-keyvault-kv2-", - // }, - // }, - // }, - //}, + { + name: "No Azure Key volumes as no RadixAzureKeyVault-s", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{}, + expectedVolumeProps: []expectedVolumeProps{}, + }, + { + name: "No Azure Key volumes as no secret names in secret object", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{{Name: "kv1"}}, + }, + { + name: "One Azure Key volume for one secret objects secret name", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{{ + Name: "kv1", + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + }}, + expectedVolumeProps: []expectedVolumeProps{ + { + expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", + expectedVolumeMountPath: "/mnt/azure-key-vault/kv1", + expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "component1-az-keyvault-kv1-", + }, + }, + }, + }, + { + name: "Multiple Azure Key volumes for each RadixAzureKeyVault", + componentName: componentName1, + azureKeyVaults: []v1.RadixAzureKeyVault{ + { + Name: "kv1", + Path: utils.StringPtr("/mnt/customPath"), + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret1", EnvVar: "SECRET_REF1"}}, + }, + { + Name: "kv2", + Items: []v1.RadixAzureKeyVaultItem{{Name: "secret2", EnvVar: "SECRET_REF2"}}, + }, + }, + expectedVolumeProps: []expectedVolumeProps{ + { + expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv1-", + expectedVolumeMountPath: "/mnt/customPath", + expectedNodePublishSecretRefName: "component1-kv1-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "component1-az-keyvault-kv1-", + }, + }, + { + expectedVolumeNamePrefix: "component1-az-keyvault-opaque-kv2-", + expectedVolumeMountPath: "/mnt/azure-key-vault/kv2", + expectedNodePublishSecretRefName: "component1-kv2-csiazkvcreds", + expectedVolumeAttributePrefixes: map[string]string{ + "secretProviderClass": "component1-az-keyvault-kv2-", + }, + }, + }, + }, { name: "Volume name should be trimmed when exceeding 63 chars", componentName: componentNameLong, @@ -114,53 +116,53 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { }, }, } - //t.Run("CSI Azure Key vault volumes", func(t *testing.T) { - // t.Parallel() - // for _, scenario := range scenarios { - // t.Logf("Test case %s", scenario.name) - // rdBuilder := getRdBuilderWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { - // var builders []utils.DeployComponentBuilder - // builders = append(builders, utils.NewDeployComponentBuilder(). - // WithName(trimComponentName(scenario.componentName)). - // WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) - // return builders - // }) - // deployment := getDeployment(t, rdBuilder) - // radixDeployComponent := deployment.radixDeployment.GetComponentByName(trimComponentName(scenario.componentName)) - // for _, azureKeyVault := range scenario.azureKeyVaults { - // spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) - // if err != nil { - // t.Log(err.Error()) - // } else { - // t.Logf("created secret provider class %s", spc.Name) - // } - // } - // volumes, err := volumemount.GetVolumes(context.Background(), deployment.kubeutil, namespace, radixDeployComponent, deployment.radixDeployment.GetName(), nil) - // require.NoError(t, err, "failed to get volumes") - // assert.Len(t, volumes, len(scenario.expectedVolumeProps)) - // if len(scenario.expectedVolumeProps) == 0 { - // continue - // } - // - // for i := 0; i < len(volumes); i++ { - // volume := volumes[i] - // assert.Less(t, len(volume.Name), 64, "volume name is too long") - // assert.NotNil(t, volume.CSI) - // assert.NotNil(t, volume.CSI.VolumeAttributes) - // assert.NotNil(t, volume.CSI.NodePublishSecretRef) - // assert.Equal(t, "secrets-store.csi.k8s.io", volume.CSI.Driver) - // - // volumeProp := scenario.expectedVolumeProps[i] - // for attrKey, attrValue := range volumeProp.expectedVolumeAttributePrefixes { - // spcValue, exists := volume.CSI.VolumeAttributes[attrKey] - // assert.True(t, exists) - // assert.True(t, strings.HasPrefix(spcValue, attrValue)) - // } - // assert.True(t, strings.Contains(volume.Name, volumeProp.expectedVolumeNamePrefix)) - // assert.Equal(t, volumeProp.expectedNodePublishSecretRefName, volume.CSI.NodePublishSecretRef.Name) - // } - // } - //}) + t.Run("CSI Azure Key vault volumes", func(t *testing.T) { + t.Parallel() + for _, scenario := range scenarios { + t.Logf("Test case %s", scenario.name) + rdBuilder := getRdBuilderWithComponentBuilders(appName, environment, func() []utils.DeployComponentBuilder { + var builders []utils.DeployComponentBuilder + builders = append(builders, utils.NewDeployComponentBuilder(). + WithName(scenario.componentName). + WithSecretRefs(v1.RadixSecretRefs{AzureKeyVaults: scenario.azureKeyVaults})) + return builders + }) + deployment := getDeployment(t, rdBuilder) + radixDeployComponent := deployment.radixDeployment.GetComponentByName(scenario.componentName) + for _, azureKeyVault := range scenario.azureKeyVaults { + spc, err := deployment.CreateAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault) + if err != nil { + t.Log(err.Error()) + } else { + t.Logf("created secret provider class %s", spc.Name) + } + } + volumes, err := volumemount.GetVolumes(context.Background(), deployment.kubeutil, namespace, radixDeployComponent, deployment.radixDeployment.GetName(), nil) + require.NoError(t, err, "failed to get volumes") + assert.Len(t, volumes, len(scenario.expectedVolumeProps)) + if len(scenario.expectedVolumeProps) == 0 { + continue + } + + for i := 0; i < len(volumes); i++ { + volume := volumes[i] + assert.Less(t, len(volume.Name), 64, "volume name is too long") + assert.NotNil(t, volume.CSI, "CSI should ne not nil") + assert.NotEmpty(t, volume.CSI.VolumeAttributes[persistentvolume.CsiVolumeSourceVolumeAttributeSecretProviderClass], "VolumeAttributes should not be empty") + assert.NotNil(t, volume.CSI.NodePublishSecretRef, "NodePublishSecretRef should not be nil") + assert.Equal(t, persistentvolume.CsiVolumeSourceDriverSecretStore, volume.CSI.Driver, "Volume driver should be %s, but it is %s", persistentvolume.CsiVolumeSourceDriverSecretStore, volume.CSI.Driver) + + volumeProp := scenario.expectedVolumeProps[i] + for attrKey, attrValue := range volumeProp.expectedVolumeAttributePrefixes { + spcValue, exists := volume.CSI.VolumeAttributes[attrKey] + assert.True(t, exists) + assert.True(t, strings.HasPrefix(spcValue, attrValue), "SecretProviderClass name should have a prefix %s, but it has name %s", attrValue, spcValue) + } + assert.True(t, strings.HasPrefix(volume.Name, volumeProp.expectedVolumeNamePrefix), "Volume name should have prefix %s, but it is %s", volumeProp.expectedVolumeNamePrefix, volume.Name) + assert.Equal(t, volumeProp.expectedNodePublishSecretRefName, volume.CSI.NodePublishSecretRef.Name, "NodePublishSecretRef name should be %s, but it is %s", volumeProp.expectedNodePublishSecretRefName, volume.CSI.NodePublishSecretRef.Name) + } + } + }) t.Run("CSI Azure Key vault volume mounts", func(t *testing.T) { //t.Parallel() @@ -195,7 +197,7 @@ func Test_CreateOrUpdateCsiAzureKeyVaultResources(t *testing.T) { volumeMount := volumeMounts[i] volumeProp := scenario.expectedVolumeProps[i] assert.Less(t, len(volumeMount.Name), 64, "volumemount name is too long") - assert.True(t, strings.Contains(volumeMount.Name, volumeProp.expectedVolumeNamePrefix), + assert.True(t, strings.HasPrefix(volumeMount.Name, volumeProp.expectedVolumeNamePrefix), "VolumeMount name should have prefix %s, but it is %s", volumeProp.expectedVolumeNamePrefix, volumeMount.Name) assert.Equal(t, volumeProp.expectedVolumeMountPath, volumeMount.MountPath, "VolumeMount mount path should be %s, but it is %s", volumeProp.expectedVolumeMountPath, volumeMount.MountPath) assert.True(t, volumeMount.ReadOnly, "VolumeMount should be read only") From 900425aae644f8e61b34adfd265b990210c5341a Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 7 Jan 2025 14:14:54 +0100 Subject: [PATCH 52/68] Reuse existing pv and pvc for re-deployed job component --- pkg/apis/batch/syncer.go | 16 +--------------- pkg/apis/deployment/kubedeployment.go | 17 ++++++++++++++++- pkg/apis/volumemount/volumemount.go | 13 +++++++++++++ 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/pkg/apis/batch/syncer.go b/pkg/apis/batch/syncer.go index ec29408b4..72f4b896f 100644 --- a/pkg/apis/batch/syncer.go +++ b/pkg/apis/batch/syncer.go @@ -6,14 +6,12 @@ import ( commonutils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/slice" "github.com/equinor/radix-operator/pkg/apis/config" - "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" "github.com/equinor/radix-operator/pkg/apis/volumemount" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" "github.com/rs/zerolog/log" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" @@ -98,7 +96,7 @@ func (s *syncer) reconcile(ctx context.Context) error { if err != nil { return err } - existingVolumes, err := getExistingVolumes(ctx, s.kubeUtil, namespace, jobComponent) + existingVolumes, err := volumemount.GetExistingJobAuxComponentVolumes(ctx, s.kubeUtil, namespace, jobComponent.GetName()) if err != nil { return err } @@ -130,18 +128,6 @@ func (s *syncer) reconcile(ctx context.Context) error { return nil } -func getExistingVolumes(ctx context.Context, kubeUtil *kube.Kube, namespace string, jobComponent *radixv1.RadixDeployJobComponent) ([]corev1.Volume, error) { - jobAuxKubeDeploymentName := defaults.GetJobAuxKubeDeployName(jobComponent.GetName()) - jobAuxKubeDeployment, err := kubeUtil.GetDeployment(ctx, namespace, jobAuxKubeDeploymentName) - if err != nil { - if errors.IsNotFound(err) { - return nil, nil - } - return nil, err - } - return jobAuxKubeDeployment.Spec.Template.Spec.Volumes, nil -} - func (s *syncer) getRadixDeploymentAndJobComponent(ctx context.Context) (*radixv1.RadixDeployment, *radixv1.RadixDeployJobComponent, error) { rd, err := s.getRadixDeployment(ctx) if err != nil { diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index d3fde3e28..0a61d04f0 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -291,7 +291,11 @@ func (deploy *Deployment) setDesiredDeploymentProperties(ctx context.Context, de desiredDeployment.Spec.Template.Spec.Affinity = utils.GetAffinityForDeployComponent(ctx, deployComponent, appName, componentName) desiredDeployment.Spec.Template.Spec.Tolerations = utils.GetDeploymentPodSpecTolerations(deployComponent.GetNode()) - volumes, err := volumemount.GetVolumes(ctx, deploy.kubeutil, deploy.getNamespace(), deployComponent, deploy.radixDeployment.GetName(), desiredDeployment.Spec.Template.Spec.Volumes) + existingVolumes, err := deploy.getDeployComponentExistingVolumes(ctx, deployComponent, desiredDeployment) + if err != nil { + return err + } + volumes, err := volumemount.GetVolumes(ctx, deploy.kubeutil, deploy.getNamespace(), deployComponent, deploy.radixDeployment.GetName(), existingVolumes) if err != nil { return err } @@ -334,6 +338,17 @@ func (deploy *Deployment) setDesiredDeploymentProperties(ctx context.Context, de return nil } +func (deploy *Deployment) getDeployComponentExistingVolumes(ctx context.Context, deployComponent v1.RadixCommonDeployComponent, deployment *appsv1.Deployment) ([]corev1.Volume, error) { + if internal.IsDeployComponentJobSchedulerDeployment(deployComponent) { + volumes, err := volumemount.GetExistingJobAuxComponentVolumes(ctx, deploy.kubeutil, deploy.getNamespace(), deployComponent.GetName()) + if err != nil { + return nil, err + } + return volumes, nil + } + return deployment.Spec.Template.Spec.Volumes, nil +} + func (deploy *Deployment) getRadixBranchAndCommitId() (string, string) { const branchKey, commitIDKey = "radix-branch", "radix-commit" rdLabels := deploy.radixDeployment.Labels diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index e387cd0b7..b95f18420 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -1022,3 +1022,16 @@ func garbageCollectSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace s } return nil } + +// GetExistingJobAuxComponentVolumes Get existing job aux component volumes +func GetExistingJobAuxComponentVolumes(ctx context.Context, kubeUtil *kube.Kube, namespace, jobComponentName string) ([]corev1.Volume, error) { + jobAuxKubeDeploymentName := defaults.GetJobAuxKubeDeployName(jobComponentName) + jobAuxKubeDeployment, err := kubeUtil.GetDeployment(ctx, namespace, jobAuxKubeDeploymentName) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return jobAuxKubeDeployment.Spec.Template.Spec.Volumes, nil +} From f87512d7594752498d795ec9e673f9823767b8ff Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 7 Jan 2025 15:47:42 +0100 Subject: [PATCH 53/68] Reuse existing pv and pvc for re-deployed job component --- .../persistentvolume/persistentvolume.go | 8 ++++-- pkg/apis/volumemount/volumemount.go | 26 +++++++++++-------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/pkg/apis/internal/persistentvolume/persistentvolume.go b/pkg/apis/internal/persistentvolume/persistentvolume.go index 11374c689..ea25e1e97 100644 --- a/pkg/apis/internal/persistentvolume/persistentvolume.go +++ b/pkg/apis/internal/persistentvolume/persistentvolume.go @@ -8,6 +8,10 @@ import ( corev1 "k8s.io/api/core/v1" ) +var ( + ignoreVolumeAttributeKeys = map[string]any{CsiVolumeMountAttributePvName: struct{}{}, CsiVolumeMountAttributePvcName: struct{}{}, CsiVolumeMountAttributePvcNamespace: struct{}{}, CsiVolumeMountAttributeProvisionerIdentity: struct{}{}} +) + // EqualPersistentVolumes Compare two PersistentVolumes func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { if pv1 == nil || pv2 == nil { @@ -140,8 +144,8 @@ func getPvAnnotations(pv *corev1.PersistentVolume) map[string]string { func getVolumeAttributes(pv *corev1.PersistentVolume) map[string]string { attributes := make(map[string]string) for key, value := range pv.Spec.CSI.VolumeAttributes { - if key == CsiVolumeMountAttributeProvisionerIdentity { - continue // ignore automatically added attribute(s) + if _, ok := ignoreVolumeAttributeKeys[key]; ok { + continue // ignore automatically added and name specific attribute(s) } attributes[key] = value } diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index b95f18420..5776b52e2 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -828,8 +828,8 @@ func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeCli } appName := radixDeployment.Spec.AppName pvcName := volume.PersistentVolumeClaim.ClaimName - pvName, actualPvExists := getActualExistingCsiAzurePvNameByPvcName(appName, namespace, componentName, radixVolumeMount, functionalPvList, pvcName, identity) - if !actualPvExists { + pvName, existsPvByPvcName, existsPvByVolumeContent := getActualExistingCsiAzurePvName(appName, namespace, componentName, radixVolumeMount, functionalPvList, pvcName, identity) + if !existsPvByPvcName && !existsPvByVolumeContent { pvName = getCsiAzurePersistentVolumeName() } existingPvc, pvcExist := pvcByNameMap[pvcName] @@ -840,7 +840,7 @@ func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeCli if pvcExist && !persistentvolume.EqualPersistentVolumeClaims(existingPvc, newPvc) { pvcName = newPvc.GetName() } - if !actualPvExists { + if !existsPvByPvcName && !existsPvByVolumeContent { log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) pv := populateCsiAzurePersistentVolume(&corev1.PersistentVolume{}, appName, namespace, componentName, pvName, pvcName, radixVolumeMount, identity) if _, err = kubeClient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil { @@ -923,16 +923,20 @@ func garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx contex return nil } -func getActualExistingCsiAzurePvNameByPvcName(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, existingPvList []corev1.PersistentVolume, pvcName string, identity *radixv1.Identity) (string, bool) { - for _, existingPv := range existingPvList { - if existingPv.Spec.ClaimRef != nil && existingPv.Spec.ClaimRef.Name == pvcName { - desiredPv := populateCsiAzurePersistentVolume(existingPv.DeepCopy(), appName, namespace, componentName, existingPv.GetName(), pvcName, radixVolumeMount, identity) - if persistentvolume.EqualPersistentVolumes(&existingPv, desiredPv) { - return existingPv.GetName(), true - } +func getActualExistingCsiAzurePvName(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumes []corev1.PersistentVolume, pvcName string, identity *radixv1.Identity) (string, bool, bool) { + for _, pv := range persistentVolumes { + if pv.Spec.ClaimRef == nil { + continue + } + if pv.Spec.ClaimRef.Name == pvcName { + return pv.GetName(), true, false + } + desiredPv := populateCsiAzurePersistentVolume(pv.DeepCopy(), appName, namespace, componentName, pv.GetName(), pvcName, radixVolumeMount, identity) + if persistentvolume.EqualPersistentVolumes(&pv, desiredPv) { + return pv.GetName(), false, true } } - return "", false + return "", false, false } func getRadixVolumeMountsByNameMap(radixDeployment *radixv1.RadixDeployment, componentName string) map[string]*radixv1.RadixVolumeMount { From bcd6e670753016ca13d3512bb9dafa0a090d3905 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Wed, 8 Jan 2025 10:52:10 +0100 Subject: [PATCH 54/68] Fixed replace of existing pv and pvc for changed volumeMount --- pkg/apis/volumemount/volumemount.go | 27 +- pkg/apis/volumemount/volumemount_test.go | 910 ++++++++++++----------- 2 files changed, 479 insertions(+), 458 deletions(-) diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index 5776b52e2..679dae852 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -924,19 +924,38 @@ func garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx contex } func getActualExistingCsiAzurePvName(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumes []corev1.PersistentVolume, pvcName string, identity *radixv1.Identity) (string, bool, bool) { + if pvName, ok := getActualCsiAzurePvNameByPvcName(appName, namespace, componentName, radixVolumeMount, persistentVolumes, pvcName, identity); ok { + return pvName, true, true + } + if pvName, ok := getFirstActualCsiAzurePvName(appName, namespace, componentName, radixVolumeMount, persistentVolumes, pvcName, identity); ok { + return pvName, false, true + } + return "", false, false +} + +func getFirstActualCsiAzurePvName(appName string, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumes []corev1.PersistentVolume, pvcName string, identity *radixv1.Identity) (string, bool) { for _, pv := range persistentVolumes { if pv.Spec.ClaimRef == nil { continue } - if pv.Spec.ClaimRef.Name == pvcName { - return pv.GetName(), true, false + desiredPv := populateCsiAzurePersistentVolume(pv.DeepCopy(), appName, namespace, componentName, pv.GetName(), pvcName, radixVolumeMount, identity) + if persistentvolume.EqualPersistentVolumes(&pv, desiredPv) { + return pv.GetName(), true } + } + return "", false +} + +func getActualCsiAzurePvNameByPvcName(appName string, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumes []corev1.PersistentVolume, pvcName string, identity *radixv1.Identity) (string, bool) { + if pv, ok := slice.FindFirst(persistentVolumes, func(pv corev1.PersistentVolume) bool { + return pv.Spec.ClaimRef != nil && pv.Spec.ClaimRef.Name == pvcName + }); ok { desiredPv := populateCsiAzurePersistentVolume(pv.DeepCopy(), appName, namespace, componentName, pv.GetName(), pvcName, radixVolumeMount, identity) if persistentvolume.EqualPersistentVolumes(&pv, desiredPv) { - return pv.GetName(), false, true + return pv.GetName(), true } } - return "", false, false + return "", false } func getRadixVolumeMountsByNameMap(radixDeployment *radixv1.RadixDeployment, componentName string) map[string]*radixv1.RadixVolumeMount { diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index 9d33cbe8e..8b5a1034e 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -495,244 +495,244 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { environment = "some-env" componentName = "some-component" ) - anotherNamespace := commonUtils.RandString(10) - anotherComponentName := commonUtils.RandString(10) + //anotherNamespace := commonUtils.RandString(10) + //anotherComponentName := commonUtils.RandString(10) var scenarios []deploymentVolumesTestScenario - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - type scenarioProperties struct { - changedNewRadixVolumeName string - changedNewRadixVolumeStorageName string - expectedVolumeName string - expectedNewSecretName string - expectedNewPvcName string - expectedNewPvName string - } - getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Update storage in existing volume name and storage", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.Name = scenarioProps.changedNewRadixVolumeName - vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) { - v.Name = scenarioProps.expectedVolumeName - }), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.ObjectMeta.Name = scenarioProps.expectedNewPvName - pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName - setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) - pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - changedNewRadixVolumeName: "volume101", - changedNewRadixVolumeStorageName: "storage101", - expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - expectedNewSecretName: "some-component-volume101-csiazurecreds", - expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - expectedNewPvName: "pv-radixvolumemount-some-uuid", - }), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) - pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) - pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) - matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - return deploymentVolumesTestScenario{ - name: "Garbage collect orphaned PVCs and PersistentVolume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - createRandomPv(props, props.namespace, props.componentName), - pvForAnotherNamespace, - pvForAnotherComponent, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - pvForAnotherNamespace, - pvForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Set readonly volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - createRandomPv(props, props.namespace, props.componentName), - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadWriteOnce volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - existingPvc, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - existingPv, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadWriteMany volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - existingPvc, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - }), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - existingPv, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // type scenarioProperties struct { + // changedNewRadixVolumeName string + // changedNewRadixVolumeStorageName string + // expectedVolumeName string + // expectedNewSecretName string + // expectedNewPvcName string + // expectedNewPvName string + // } + // getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Update storage in existing volume name and storage", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.Name = scenarioProps.changedNewRadixVolumeName + // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) { + // v.Name = scenarioProps.expectedVolumeName + // }), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName + // setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + // pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + // changedNewRadixVolumeName: "volume101", + // changedNewRadixVolumeStorageName: "storage101", + // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + // expectedNewSecretName: "some-component-volume101-csiazurecreds", + // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + // expectedNewPvName: "pv-radixvolumemount-some-uuid", + // }), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) + // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + // pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) + // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + // matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // return deploymentVolumesTestScenario{ + // name: "Garbage collect orphaned PVCs and PersistentVolume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPv(props, props.namespace, props.componentName), + // pvForAnotherNamespace, + // pvForAnotherComponent, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // pvForAnotherNamespace, + // pvForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Set readonly volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // createRandomPv(props, props.namespace, props.componentName), + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteOnce volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // existingPvc, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteMany volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // existingPvc, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // }), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { existingPvc := createExpectedPvc(props, nil) @@ -775,222 +775,222 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { getScenario(getPropsCsiBlobVolume1Storage1(nil)), } }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, - "--streaming=true", - "--stream-cache-mb=101", - "--block-size-mb=102", - "--buffer-size-mb=103", - "--max-buffers=104", - "--max-blocks-per-file=105", - "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has disabled streaming", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - Enabled: pointers.Ptr(false), - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, - "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) - pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) - matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) - return deploymentVolumesTestScenario{ - name: "Do not change existing PersistentVolume with class name, when creating new PVC", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - pvcForAnotherComponent, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - pvcForAnotherComponent, - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - existingPv, - pvForAnotherComponent, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - existingPv, - pvForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) - return deploymentVolumesTestScenario{ - name: "Do not change existing PersistentVolume without class name, when creating new PVC", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - pvcForAnotherComponent, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - pvcForAnotherComponent, - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - existingPv, - pvForAnotherComponent, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - existingPv, - pvForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) - pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) - matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) - expectedPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) - return deploymentVolumesTestScenario{ - name: "Do not change existing PVC with class name, when creating new PersistentVolume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - pvcForAnotherComponent, - existingPvc, - }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - existingPvc, - pvcForAnotherComponent, - }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ - pvForAnotherComponent, - }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ - expectedPv, - pvForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + // StreamCache: pointers.Ptr(uint64(101)), + // BlockSize: pointers.Ptr(uint64(102)), + // BufferSize: pointers.Ptr(uint64(103)), + // MaxBuffers: pointers.Ptr(uint64(104)), + // MaxBlocksPerFile: pointers.Ptr(uint64(105)), + // } + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, + // "--streaming=true", + // "--stream-cache-mb=101", + // "--block-size-mb=102", + // "--buffer-size-mb=103", + // "--max-buffers=104", + // "--max-blocks-per-file=105", + // "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) + // + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has disabled streaming", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + // Enabled: pointers.Ptr(false), + // StreamCache: pointers.Ptr(uint64(101)), + // BlockSize: pointers.Ptr(uint64(102)), + // BufferSize: pointers.Ptr(uint64(103)), + // MaxBuffers: pointers.Ptr(uint64(104)), + // MaxBlocksPerFile: pointers.Ptr(uint64(105)), + // } + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, + // "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) + // pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) + // return deploymentVolumesTestScenario{ + // name: "Do not change existing PersistentVolume with class name, when creating new PVC", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // pvcForAnotherComponent, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherComponent, + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + // return deploymentVolumesTestScenario{ + // name: "Do not change existing PersistentVolume without class name, when creating new PVC", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // pvcForAnotherComponent, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherComponent, + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) + // pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) + // expectedPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + // return deploymentVolumesTestScenario{ + // name: "Do not change existing PVC with class name, when creating new PersistentVolume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) {}), + // }, + // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + // pvcForAnotherComponent, + // existingPvc, + // }, + // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + // existingPvc, + // pvcForAnotherComponent, + // }, + // existingPVsBeforeTestRun: []corev1.PersistentVolume{ + // pvForAnotherComponent, + // }, + // existingPVsAfterTestRun: []corev1.PersistentVolume{ + // expectedPv, + // pvForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { for _, factory := range suite.radixCommonDeployComponentFactories { @@ -998,7 +998,9 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) testEnv := getTestEnv() radixDeployment := buildRd(appName, environment, componentName, scenario.radixVolumeMounts) - putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeclient, &scenario) + putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeUtil.KubeClient(), &scenario) + existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) + assert.Nil(t, err) desiredVolumes := getDesiredDeployment(componentName, scenario.volumes).Spec.Template.Spec.Volumes deployComponent := radixDeployment.Spec.Components[0] @@ -1006,7 +1008,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { assert.Nil(t, err) assert.Equal(t, len(scenario.volumes), len(actualVolumes), "Number of volumes is not equal") - existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) + existingPvcs, existingPvs, err = getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) assert.Nil(t, err) assert.True(t, equalPersistentVolumeClaims(&scenario.existingPvcsAfterTestRun, &existingPvcs), "PVC-s are not equal") assert.True(t, equalPersistentVolumes(&scenario.existingPVsAfterTestRun, &existingPvs), "PV-s are not equal") From 8a896d4de544a4e32a623be3b5657b866e6315e0 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Wed, 8 Jan 2025 16:44:04 +0100 Subject: [PATCH 55/68] Fixed replace of existing pv and pvc for changed volumeMount in job aux --- pkg/apis/deployment/deployment.go | 3 ++- pkg/apis/deployment/kubedeployment.go | 1 - pkg/apis/volumemount/volumemount.go | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 250c0963f..5c87eed8c 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -621,7 +621,8 @@ func syncRadixRestartEnvironmentVariable(deployComponent v1.RadixCommonDeployCom func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, namespace, jobKubeDeploymentName string) (*appsv1.Deployment, *appsv1.Deployment, error) { jobAuxKubeDeploymentName := defaults.GetJobAuxKubeDeployName(jobKubeDeploymentName) - currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(ctx, namespace, jobAuxKubeDeploymentName) + currentJobAuxDeployment, err := deploy.kubeutil.KubeClient().AppsV1().Deployments(namespace).Get(ctx, jobAuxKubeDeploymentName, metav1.GetOptions{}) + //currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(ctx, namespace, jobAuxKubeDeploymentName) if err != nil { if k8sErrors.IsNotFound(err) { return nil, deploy.createJobAuxDeployment(jobKubeDeploymentName, jobAuxKubeDeploymentName), nil diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index 0a61d04f0..e36815b25 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -46,7 +46,6 @@ func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployCompone desiredDeployment.Spec.Template.Spec.Volumes = actualVolumes desiredVolumeMounts := desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts if err = deploy.handleJobAuxDeployment(ctx, namespace, deployComponent, desiredDeployment, actualVolumes, desiredVolumeMounts); err != nil { - //!!!! error "failed to create Deployment object: deployments.apps \"job1-aux\" already exists" return err } return deploy.kubeutil.ApplyDeployment(ctx, namespace, currentDeployment, desiredDeployment) diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index 679dae852..9386d6949 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -837,7 +837,8 @@ func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeCli if err != nil { return nil, err } - if pvcExist && !persistentvolume.EqualPersistentVolumeClaims(existingPvc, newPvc) { + if pvcExist && + (!persistentvolume.EqualPersistentVolumeClaims(existingPvc, newPvc) || existingPvc.Spec.VolumeName != pvName) { pvcName = newPvc.GetName() } if !existsPvByPvcName && !existsPvByVolumeContent { From a27e812feed919d5ee4c64bdd2733735b796df6a Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 9 Jan 2025 14:10:30 +0100 Subject: [PATCH 56/68] Set version. Cleanup --- charts/radix-operator/Chart.yaml | 4 ++-- pipeline-runner/internal/test/utils.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/radix-operator/Chart.yaml b/charts/radix-operator/Chart.yaml index f11a12a64..79eb1440e 100644 --- a/charts/radix-operator/Chart.yaml +++ b/charts/radix-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: radix-operator -version: 1.48.3 -appVersion: 1.68.3 +version: 1.49.0 +appVersion: 1.69.0 kubeVersion: ">=1.24.0" description: Radix Operator keywords: diff --git a/pipeline-runner/internal/test/utils.go b/pipeline-runner/internal/test/utils.go index e84050e42..9ab61bd05 100644 --- a/pipeline-runner/internal/test/utils.go +++ b/pipeline-runner/internal/test/utils.go @@ -8,7 +8,7 @@ import ( "github.com/equinor/radix-operator/pkg/apis/defaults" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" - hash2 "github.com/equinor/radix-operator/pkg/apis/utils/hash" + "github.com/equinor/radix-operator/pkg/apis/utils/hash" "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -64,18 +64,18 @@ func CreateBuildSecret(kubeClient kubernetes.Interface, appName string, data map func GetRadixApplicationHash(ra *radixv1.RadixApplication) string { if ra == nil { - hash, _ := hash2.ToHashString(hash2.SHA256, "0nXSg9l6EUepshGFmolpgV3elB0m8Mv7") + hash, _ := hash.ToHashString(hash.SHA256, "0nXSg9l6EUepshGFmolpgV3elB0m8Mv7") return hash } - hash, _ := hash2.ToHashString(hash2.SHA256, ra.Spec) + hash, _ := hash.ToHashString(hash.SHA256, ra.Spec) return hash } func GetBuildSecretHash(secret *corev1.Secret) string { if secret == nil { - hash, _ := hash2.ToHashString(hash2.SHA256, "34Wd68DsJRUzrHp2f63o3U5hUD6zl8Tj") + hash, _ := hash.ToHashString(hash.SHA256, "34Wd68DsJRUzrHp2f63o3U5hUD6zl8Tj") return hash } - hash, _ := hash2.ToHashString(hash2.SHA256, secret.Data) + hash, _ := hash.ToHashString(hash.SHA256, secret.Data) return hash } From 2c5528544833468cf729846eac5a86234f747207 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 9 Jan 2025 14:22:22 +0100 Subject: [PATCH 57/68] Cleanup --- pkg/apis/batch/kubejob.go | 2 +- pkg/apis/batch/syncer.go | 1 + pkg/apis/deployment/deployment.go | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/apis/batch/kubejob.go b/pkg/apis/batch/kubejob.go index 127f0b868..b02b8c4b6 100644 --- a/pkg/apis/batch/kubejob.go +++ b/pkg/apis/batch/kubejob.go @@ -3,7 +3,6 @@ package batch import ( "context" "fmt" - "github.com/equinor/radix-operator/pkg/apis/volumemount" "strings" "github.com/equinor/radix-common/utils/numbers" @@ -16,6 +15,7 @@ import ( operatorUtils "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/equinor/radix-operator/pkg/apis/utils/annotations" radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" + "github.com/equinor/radix-operator/pkg/apis/volumemount" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/apis/batch/syncer.go b/pkg/apis/batch/syncer.go index 72f4b896f..7eb1d3571 100644 --- a/pkg/apis/batch/syncer.go +++ b/pkg/apis/batch/syncer.go @@ -3,6 +3,7 @@ package batch import ( "context" "fmt" + commonutils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/slice" "github.com/equinor/radix-operator/pkg/apis/config" diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 5c87eed8c..620df1918 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -622,7 +622,6 @@ func syncRadixRestartEnvironmentVariable(deployComponent v1.RadixCommonDeployCom func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Context, namespace, jobKubeDeploymentName string) (*appsv1.Deployment, *appsv1.Deployment, error) { jobAuxKubeDeploymentName := defaults.GetJobAuxKubeDeployName(jobKubeDeploymentName) currentJobAuxDeployment, err := deploy.kubeutil.KubeClient().AppsV1().Deployments(namespace).Get(ctx, jobAuxKubeDeploymentName, metav1.GetOptions{}) - //currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(ctx, namespace, jobAuxKubeDeploymentName) if err != nil { if k8sErrors.IsNotFound(err) { return nil, deploy.createJobAuxDeployment(jobKubeDeploymentName, jobAuxKubeDeploymentName), nil From 8f10ac61eccd09fbdb820612cbd6adff5e60586f Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 10 Jan 2025 09:44:58 +0100 Subject: [PATCH 58/68] Uncommented unit-tests, fixed linter reported issues --- pkg/apis/deployment/secrets.go | 12 - .../persistentvolume/persistentvolume.go | 3 +- .../persistentvolume/persistentvolumeclaim.go | 5 +- pkg/apis/volumemount/volumemount.go | 46 +- pkg/apis/volumemount/volumemount_test.go | 903 +++++++++--------- 5 files changed, 440 insertions(+), 529 deletions(-) diff --git a/pkg/apis/deployment/secrets.go b/pkg/apis/deployment/secrets.go index 3bbb90075..1c324f83c 100644 --- a/pkg/apis/deployment/secrets.go +++ b/pkg/apis/deployment/secrets.go @@ -109,18 +109,6 @@ func (deploy *Deployment) createOrUpdateSecretsForComponent(ctx context.Context, return nil } -func (deploy *Deployment) getBlobFuseCredsSecrets(ctx context.Context, ns, componentName, volumeMountName string) (string, []byte, []byte) { - secretName := defaults.GetBlobFuseCredsSecretName(componentName, volumeMountName) - accountKey := []byte(defaults.SecretDefaultData) - accountName := []byte(defaults.SecretDefaultData) - if deploy.kubeutil.SecretExists(ctx, ns, secretName) { - oldSecret, _ := deploy.kubeutil.GetSecret(ctx, ns, secretName) - accountKey = oldSecret.Data[defaults.BlobFuseCredsAccountKeyPart] - accountName = oldSecret.Data[defaults.BlobFuseCredsAccountNamePart] - } - return secretName, accountKey, accountName -} - func (deploy *Deployment) garbageCollectSecretsNoLongerInSpec(ctx context.Context) error { secrets, err := deploy.kubeutil.ListSecrets(ctx, deploy.radixDeployment.GetNamespace()) if err != nil { diff --git a/pkg/apis/internal/persistentvolume/persistentvolume.go b/pkg/apis/internal/persistentvolume/persistentvolume.go index ea25e1e97..6abfcc5a5 100644 --- a/pkg/apis/internal/persistentvolume/persistentvolume.go +++ b/pkg/apis/internal/persistentvolume/persistentvolume.go @@ -45,7 +45,8 @@ func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { return false } if pv1.Spec.ClaimRef != nil { - if pv2.Spec.ClaimRef == nil || pv1.Spec.ClaimRef.Name != pv2.Spec.ClaimRef.Name { + if pv2.Spec.ClaimRef == nil || len(pv1.Spec.ClaimRef.Name) < 5 || len(pv2.Spec.ClaimRef.Name) < 5 || + pv1.Spec.ClaimRef.Name[:len(pv1.Spec.ClaimRef.Name)-5] != pv2.Spec.ClaimRef.Name[:len(pv2.Spec.ClaimRef.Name)-5] { return false } } else if pv2.Spec.ClaimRef != nil { diff --git a/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go b/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go index 47111e04e..31125d48a 100644 --- a/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go +++ b/pkg/apis/internal/persistentvolume/persistentvolumeclaim.go @@ -46,10 +46,7 @@ func EqualPersistentVolumeClaims(pvc1, pvc2 *corev1.PersistentVolumeClaim) bool } volumeMode1 := pointers.Val(pvc1.Spec.VolumeMode) volumeMode2 := pointers.Val(pvc2.Spec.VolumeMode) - if volumeMode1 != volumeMode2 { - return false - } - return true + return volumeMode1 == volumeMode2 } func getAnnotations(pvc *corev1.PersistentVolumeClaim) map[string]string { diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index 9386d6949..dcc3604ff 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "sort" "strings" commonUtils "github.com/equinor/radix-common/utils" @@ -41,9 +40,8 @@ const ( ) var ( - csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}} - functionalPersistentVolumePhases = []corev1.PersistentVolumePhase{corev1.VolumePending, corev1.VolumeBound, corev1.VolumeAvailable} - functionalPersistentVolumeClaimPhases = []corev1.PersistentVolumeClaimPhase{corev1.ClaimPending, corev1.ClaimBound} + csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}} + functionalPersistentVolumePhases = []corev1.PersistentVolumePhase{corev1.VolumePending, corev1.VolumeBound, corev1.VolumeAvailable} ) // GetRadixDeployComponentVolumeMounts Gets list of v1.VolumeMount for radixv1.RadixCommonDeployComponent @@ -397,21 +395,6 @@ func getVolumeMountDeprecatedVolumeName(volumeMount *radixv1.RadixVolumeMount, c return "", fmt.Errorf("unsupported volume type %s", volumeMount.Type) } -func getFunctionalPvcList(ctx context.Context, kubeclient kubernetes.Interface, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) ([]corev1.PersistentVolumeClaim, error) { - pvcList, err := kubeclient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentVolumeMount(componentName, radixVolumeMount.Name), - }) - if err != nil { - return nil, err - } - existingPvcs := sortPvcsByCreatedTimestampDesc(pvcList.Items) - return slice.FindAll(existingPvcs, func(pvc corev1.PersistentVolumeClaim) bool { return pvcIsFunctional(pvc) }), nil -} - -func pvcIsFunctional(pvc corev1.PersistentVolumeClaim) bool { - return slice.Any(functionalPersistentVolumeClaimPhases, func(phase corev1.PersistentVolumeClaimPhase) bool { return pvc.Status.Phase == phase }) -} - func getCsiAzurePersistentVolumeClaimName(componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { volumeName, err := getCsiAzureVolumeMountName(componentName, radixVolumeMount) if err != nil { @@ -475,10 +458,6 @@ func getLabelSelectorForCsiAzurePersistenceVolumeClaim(componentName string) str return fmt.Sprintf("%s=%s, %s in (%s, %s)", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure)) } -func getLabelSelectorForCsiAzurePersistenceVolumeClaimForComponentVolumeMount(componentName, radixVolumeMountName string) string { - return fmt.Sprintf("%s=%s, %s=%s", kube.RadixComponentLabel, componentName, kube.RadixVolumeMountNameLabel, radixVolumeMountName) -} - func buildPersistentVolumeClaim(appName, namespace, componentName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { pvcName, err := getCsiAzurePersistentVolumeClaimName(componentName, radixVolumeMount) if err != nil { @@ -696,13 +675,6 @@ func getRadixBlobFuse2VolumeMountGid(radixVolumeMount *radixv1.RadixVolumeMount) return radixVolumeMount.GID } -func getRadixBlobFuse2VolumeMountSkuName(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.BlobFuse2 != nil { - return radixVolumeMount.BlobFuse2.SkuName - } - return radixVolumeMount.SkuName -} - func getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount *radixv1.RadixVolumeMount) string { if radixVolumeMount.BlobFuse2 != nil { return radixVolumeMount.BlobFuse2.Container @@ -717,13 +689,6 @@ func getRadixBlobFuse2VolumeMountRequestsStorage(radixVolumeMount *radixv1.Radix return radixVolumeMount.RequestsStorage } -func getRadixBlobFuse2VolumeMountBindingMode(radixVolumeMount *radixv1.RadixVolumeMount) string { - if radixVolumeMount.BlobFuse2 != nil { - return radixVolumeMount.BlobFuse2.BindingMode - } - return radixVolumeMount.BindingMode -} - func deletePersistentVolumeClaim(ctx context.Context, kubeClient kubernetes.Interface, namespace, pvcName string) error { if len(namespace) == 0 || len(pvcName) == 0 { log.Ctx(ctx).Debug().Msgf("Skip deleting PVC - namespace %s or name %s is empty", namespace, pvcName) @@ -992,13 +957,6 @@ func findCsiAzureVolumeForComponent(volumeMountsByNameMap map[string]*radixv1.Ra return true } -func sortPvcsByCreatedTimestampDesc(persistentVolumeClaims []corev1.PersistentVolumeClaim) []corev1.PersistentVolumeClaim { - sort.SliceStable(persistentVolumeClaims, func(i, j int) bool { - return (persistentVolumeClaims)[j].ObjectMeta.CreationTimestamp.Before(&(persistentVolumeClaims)[i].ObjectMeta.CreationTimestamp) - }) - return persistentVolumeClaims -} - func trimVolumeNameToValidLength(volumeName string) string { const randSize = 5 if len(volumeName) <= volumeNameMaxLength { diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index 8b5a1034e..74a294f77 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -2,7 +2,6 @@ package volumemount import ( "context" - "encoding/json" "fmt" "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/google/uuid" @@ -30,7 +29,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes" kubefake "k8s.io/client-go/kubernetes/fake" secretProviderClient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" @@ -495,244 +493,244 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { environment = "some-env" componentName = "some-component" ) - //anotherNamespace := commonUtils.RandString(10) - //anotherComponentName := commonUtils.RandString(10) + anotherNamespace := commonUtils.RandString(10) + anotherComponentName := commonUtils.RandString(10) var scenarios []deploymentVolumesTestScenario - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // type scenarioProperties struct { - // changedNewRadixVolumeName string - // changedNewRadixVolumeStorageName string - // expectedVolumeName string - // expectedNewSecretName string - // expectedNewPvcName string - // expectedNewPvName string - // } - // getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Update storage in existing volume name and storage", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.Name = scenarioProps.changedNewRadixVolumeName - // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) { - // v.Name = scenarioProps.expectedVolumeName - // }), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName - // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName - // setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) - // pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - // changedNewRadixVolumeName: "volume101", - // changedNewRadixVolumeStorageName: "storage101", - // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - // expectedNewSecretName: "some-component-volume101-csiazurecreds", - // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - // expectedNewPvName: "pv-radixvolumemount-some-uuid", - // }), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) - // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - // pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) - // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - // matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // return deploymentVolumesTestScenario{ - // name: "Garbage collect orphaned PVCs and PersistentVolume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPv(props, props.namespace, props.componentName), - // pvForAnotherNamespace, - // pvForAnotherComponent, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // pvForAnotherNamespace, - // pvForAnotherComponent, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Set readonly volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // createRandomPv(props, props.namespace, props.componentName), - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteOnce volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // existingPvc, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteMany volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // existingPvc, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // }), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + type scenarioProperties struct { + changedNewRadixVolumeName string + changedNewRadixVolumeStorageName string + expectedVolumeName string + expectedNewSecretName string + expectedNewPvcName string + expectedNewPvName string + } + getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Update storage in existing volume name and storage", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.Name = scenarioProps.changedNewRadixVolumeName + vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) { + v.Name = scenarioProps.expectedVolumeName + }), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName + setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + changedNewRadixVolumeName: "volume101", + changedNewRadixVolumeStorageName: "storage101", + expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + expectedNewSecretName: "some-component-volume101-csiazurecreds", + expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + expectedNewPvName: "pv-radixvolumemount-some-uuid", + }), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) + pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) + pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + return deploymentVolumesTestScenario{ + name: "Garbage collect orphaned PVCs and PersistentVolume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createRandomPvc(props, props.namespace, props.componentName), + pvcForAnotherNamespace, + pvcForAnotherComponent, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherNamespace, + pvcForAnotherComponent, + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createRandomPv(props, props.namespace, props.componentName), + pvForAnotherNamespace, + pvForAnotherComponent, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + pvForAnotherNamespace, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Set readonly volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + createRandomPvc(props, props.namespace, props.componentName), + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + createRandomPv(props, props.namespace, props.componentName), + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteOnce volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteMany volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { existingPvc := createExpectedPvc(props, nil) @@ -775,185 +773,187 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { getScenario(getPropsCsiBlobVolume1Storage1(nil)), } }()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - // StreamCache: pointers.Ptr(uint64(101)), - // BlockSize: pointers.Ptr(uint64(102)), - // BufferSize: pointers.Ptr(uint64(103)), - // MaxBuffers: pointers.Ptr(uint64(104)), - // MaxBlocksPerFile: pointers.Ptr(uint64(105)), - // } - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, - // "--streaming=true", - // "--stream-cache-mb=101", - // "--block-size-mb=102", - // "--buffer-size-mb=103", - // "--max-buffers=104", - // "--max-blocks-per-file=105", - // "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) - // - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has disabled streaming", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - // Enabled: pointers.Ptr(false), - // StreamCache: pointers.Ptr(uint64(101)), - // BlockSize: pointers.Ptr(uint64(102)), - // BufferSize: pointers.Ptr(uint64(103)), - // MaxBuffers: pointers.Ptr(uint64(104)), - // MaxBlocksPerFile: pointers.Ptr(uint64(105)), - // } - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, - // "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) - // pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) - // return deploymentVolumesTestScenario{ - // name: "Do not change existing PersistentVolume with class name, when creating new PVC", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // pvcForAnotherComponent, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherComponent, - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) - // return deploymentVolumesTestScenario{ - // name: "Do not change existing PersistentVolume without class name, when creating new PVC", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // pvcForAnotherComponent, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherComponent, - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, + "--streaming=true", + "--stream-cache-mb=101", + "--block-size-mb=102", + "--buffer-size-mb=103", + "--max-buffers=104", + "--max-blocks-per-file=105", + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has disabled streaming", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + Enabled: pointers.Ptr(false), + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{}, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) + pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) + return deploymentVolumesTestScenario{ + name: "Do not change existing PersistentVolume with class name, when creating new PVC", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherComponent, + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + return deploymentVolumesTestScenario{ + name: "Do not change existing PersistentVolume without class name, when creating new PVC", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + }, + existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherComponent, + }, + existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + existingPVsAfterTestRun: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + // TODO fixing it //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { // pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) @@ -961,6 +961,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) // existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) // expectedPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + // matchPvAndPvc(&expectedPv, &existingPvc) // return deploymentVolumesTestScenario{ // name: "Do not change existing PVC with class name, when creating new PersistentVolume", // props: props, @@ -975,15 +976,15 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // existingPvc, // }, // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // existingPvc, // pvcForAnotherComponent, + // existingPvc, // }, // existingPVsBeforeTestRun: []corev1.PersistentVolume{ // pvForAnotherComponent, // }, // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // expectedPv, // pvForAnotherComponent, + // expectedPv, // }, // } // } @@ -999,17 +1000,15 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { testEnv := getTestEnv() radixDeployment := buildRd(appName, environment, componentName, scenario.radixVolumeMounts) putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeUtil.KubeClient(), &scenario) - existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) - assert.Nil(t, err) desiredVolumes := getDesiredDeployment(componentName, scenario.volumes).Spec.Template.Spec.Volumes deployComponent := radixDeployment.Spec.Components[0] actualVolumes, err := CreateOrUpdateCsiAzureVolumeResources(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, utils.GetEnvironmentNamespace(appName, environment), &deployComponent, desiredVolumes) - assert.Nil(t, err) + require.NoError(t, err) assert.Equal(t, len(scenario.volumes), len(actualVolumes), "Number of volumes is not equal") - existingPvcs, existingPvs, err = getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) - assert.Nil(t, err) + existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) + require.NoError(t, err) assert.True(t, equalPersistentVolumeClaims(&scenario.existingPvcsAfterTestRun, &existingPvcs), "PVC-s are not equal") assert.True(t, equalPersistentVolumes(&scenario.existingPVsAfterTestRun, &existingPvs), "PV-s are not equal") } @@ -1206,14 +1205,6 @@ func createPvc(namespace, componentName string, mountType v1.MountType, modify f return pvc } -func buildRdWithComponentBuilders(appName string, environment string, componentBuilders func() []utils.DeployComponentBuilder) *v1.RadixDeployment { - return utils.ARadixDeployment(). - WithAppName(appName). - WithEnvironment(environment). - WithComponents(componentBuilders()...). - BuildRD() -} - func createExpectedPv(props expectedPvcPvProperties, modify func(pv *corev1.PersistentVolume)) corev1.PersistentVolume { mountOptions := getMountOptions(props, true) pv := corev1.PersistentVolume{ @@ -1526,8 +1517,8 @@ func equalPersistentVolumeClaims(pvcList1, pvcList2 *[]corev1.PersistentVolumeCl for _, pvc1 := range *pvcList1 { var hasEqualPvc bool for _, pvc2 := range *pvcList2 { - if equalNamePrefix(pvc1, pvc2) && - equalVolumeNamePrefix(pvc1, pvc2) && + if equalTillPostfix(pvc1.GetName(), pvc2.GetName(), 5) && + equalPrefix(pvc1.Spec.VolumeName, pvc2.Spec.VolumeName, 20) && persistentvolume.EqualPersistentVolumeClaims(&pvc1, &pvc2) { hasEqualPvc = true break @@ -1540,42 +1531,18 @@ func equalPersistentVolumeClaims(pvcList1, pvcList2 *[]corev1.PersistentVolumeCl return true } -func equalVolumeNamePrefix(pvc1 corev1.PersistentVolumeClaim, pvc2 corev1.PersistentVolumeClaim) bool { - prefix1 := pvc1.Spec.VolumeName[:20] - prefix2 := pvc2.Spec.VolumeName[:20] - return prefix1 == prefix2 -} - -func equalNamePrefix(pvc1 corev1.PersistentVolumeClaim, pvc2 corev1.PersistentVolumeClaim) bool { - prefix1 := pvc1.GetName()[:len(pvc1.GetName())-5] - prefix2 := pvc2.GetName()[:len(pvc2.GetName())-5] - return prefix1 == prefix2 -} - -func getPvcCopyWithLabels(pvc *corev1.PersistentVolumeClaim, ignoreRandomPostfixInName bool) (*corev1.PersistentVolumeClaim, map[string]string) { - pvcCopy := pvc.DeepCopy() - pvcCopy.ObjectMeta.ManagedFields = nil // HACK: to avoid ManagedFields comparison - if ignoreRandomPostfixInName { - pvcCopy.ObjectMeta.Name = commonUtils.ShortenString(pvcCopy.ObjectMeta.Name, 6) +func equalPrefix(value1, value2 string, prefixLength int) bool { + if len(value1) < prefixLength || len(value2) < prefixLength { + return false } - // to avoid label order variations - labels := pvcCopy.ObjectMeta.Labels - pvcCopy.ObjectMeta.Labels = map[string]string{} - return pvcCopy, labels + eq := value1[:prefixLength] == value2[:prefixLength] + return eq } -func getPvcPatch(pvc1, pvc2 *corev1.PersistentVolumeClaim) ([]byte, error) { - json1, err := json.Marshal(pvc1) - if err != nil { - return nil, err - } - json2, err := json.Marshal(pvc2) - if err != nil { - return nil, err - } - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(json1, json2, corev1.PersistentVolumeClaim{}) - if err != nil { - return nil, err +func equalTillPostfix(value1, value2 string, postfixLength int) bool { + if len(value1) < postfixLength || len(value2) < postfixLength { + return false } - return patchBytes, nil + eq := value1[:len(value1)-postfixLength] == value2[:len(value2)-postfixLength] + return eq } From e9232017255e2d6d1d54711b417f1a7d04ff441c Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 10 Jan 2025 13:37:53 +0100 Subject: [PATCH 59/68] Fixed unit-tests --- .../persistentvolume/persistentvolume.go | 97 ++----- pkg/apis/internal/strings.go | 9 + pkg/apis/volumemount/volumemount.go | 6 +- pkg/apis/volumemount/volumemount_test.go | 252 +++++++++--------- 4 files changed, 157 insertions(+), 207 deletions(-) create mode 100644 pkg/apis/internal/strings.go diff --git a/pkg/apis/internal/persistentvolume/persistentvolume.go b/pkg/apis/internal/persistentvolume/persistentvolume.go index 6abfcc5a5..1e273bc86 100644 --- a/pkg/apis/internal/persistentvolume/persistentvolume.go +++ b/pkg/apis/internal/persistentvolume/persistentvolume.go @@ -1,6 +1,7 @@ package persistentvolume import ( + "github.com/equinor/radix-operator/pkg/apis/internal" "strings" "github.com/equinor/radix-common/utils/slice" @@ -8,35 +9,35 @@ import ( corev1 "k8s.io/api/core/v1" ) -var ( - ignoreVolumeAttributeKeys = map[string]any{CsiVolumeMountAttributePvName: struct{}{}, CsiVolumeMountAttributePvcName: struct{}{}, CsiVolumeMountAttributePvcNamespace: struct{}{}, CsiVolumeMountAttributeProvisionerIdentity: struct{}{}} -) - // EqualPersistentVolumes Compare two PersistentVolumes func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { if pv1 == nil || pv2 == nil { return false } - // Ignore for now, due to during transition period this would affect existing volume mounts, managed by a provisioner + // Ignore for now, due to during transition period this would affect existing volume mounts, managed by a provisioner. When all volume mounts gets labels, uncomment these lines // if !utils.EqualStringMaps(pv1.GetLabels(), pv2.GetLabels()) { // return false, nil // } if !utils.EqualStringMaps(getPvAnnotations(pv1), getPvAnnotations(pv2)) { return false } - if !utils.EqualStringMaps(getVolumeAttributes(pv1), getVolumeAttributes(pv2)) { + expectedClonedVolumeAttrs := cloneMap(pv1.Spec.CSI.VolumeAttributes, CsiVolumeMountAttributePvName, CsiVolumeMountAttributePvcName, CsiVolumeMountAttributeProvisionerIdentity) + actualClonedVolumeAttrs := cloneMap(pv2.Spec.CSI.VolumeAttributes, CsiVolumeMountAttributePvName, CsiVolumeMountAttributePvcName, CsiVolumeMountAttributeProvisionerIdentity) + if !utils.EqualStringMaps(expectedClonedVolumeAttrs, actualClonedVolumeAttrs) { return false } if !utils.EqualStringMaps(getMountOptionsMap(pv1.Spec.MountOptions), getMountOptionsMap(pv2.Spec.MountOptions)) { return false } - // ignore pv1.Spec.StorageClassName != pv2.Spec.StorageClassName for transition period + + // ignore pv1.Spec.StorageClassName != pv2.Spec.StorageClassName for transition period. When there is no volumeMounts with storage class exists - uncomment this line if pv1.Spec.Capacity[corev1.ResourceStorage] != pv2.Spec.Capacity[corev1.ResourceStorage] || len(pv1.Spec.AccessModes) != len(pv2.Spec.AccessModes) || (len(pv1.Spec.AccessModes) != 1 && pv1.Spec.AccessModes[0] != pv2.Spec.AccessModes[0]) || pv1.Spec.CSI.Driver != pv2.Spec.CSI.Driver { return false } + if pv1.Spec.CSI.NodeStageSecretRef != nil { if pv2.Spec.CSI.NodeStageSecretRef == nil || pv1.Spec.CSI.NodeStageSecretRef.Name != pv2.Spec.CSI.NodeStageSecretRef.Name { return false @@ -44,9 +45,12 @@ func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { } else if pv2.Spec.CSI.NodeStageSecretRef != nil { return false } + if pv1.Spec.ClaimRef != nil { - if pv2.Spec.ClaimRef == nil || len(pv1.Spec.ClaimRef.Name) < 5 || len(pv2.Spec.ClaimRef.Name) < 5 || - pv1.Spec.ClaimRef.Name[:len(pv1.Spec.ClaimRef.Name)-5] != pv2.Spec.ClaimRef.Name[:len(pv2.Spec.ClaimRef.Name)-5] { + if pv2.Spec.ClaimRef == nil || + !internal.EqualTillPostfix(pv1.Spec.ClaimRef.Name, pv2.Spec.ClaimRef.Name, 5) || + pv1.Spec.ClaimRef.Namespace != pv2.Spec.ClaimRef.Namespace || + pv1.Spec.ClaimRef.Kind != pv2.Spec.ClaimRef.Kind { return false } } else if pv2.Spec.ClaimRef != nil { @@ -55,73 +59,15 @@ func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { return true } -// EqualPersistentVolumesForTest Compare two PersistentVolumes for test -func EqualPersistentVolumesForTest(expectedPv, actualPv *corev1.PersistentVolume) bool { - if expectedPv == nil || actualPv == nil { - return false - } - // Ignore for now, due to during transition period this would affect existing volume mounts, managed by a provisioner - // if !utils.EqualStringMaps(expectedPv.GetLabels(), actualPv.GetLabels()) { - // return false, nil - // } - if !utils.EqualStringMaps(getPvAnnotations(expectedPv), getPvAnnotations(actualPv)) { - return false - } - expectedClonedAttrs := cloneMap(expectedPv.Spec.CSI.VolumeAttributes, CsiVolumeMountAttributePvName, CsiVolumeMountAttributePvcName, CsiVolumeMountAttributeProvisionerIdentity) - actualClonedAttrs := cloneMap(actualPv.Spec.CSI.VolumeAttributes, CsiVolumeMountAttributePvName, CsiVolumeMountAttributePvcName, CsiVolumeMountAttributeProvisionerIdentity) - if !utils.EqualStringMaps(expectedClonedAttrs, actualClonedAttrs) { - return false - } - expectedNameAttr := expectedPv.Spec.CSI.VolumeAttributes[CsiVolumeMountAttributePvName] - actualNameAttr := actualPv.Spec.CSI.VolumeAttributes[CsiVolumeMountAttributePvName] - if len(expectedNameAttr) == 0 || len(actualNameAttr) == 0 { - return false - } - if expectedNameAttr[:20] != actualNameAttr[:20] { - return false - } - expectedPvcNameAttr := expectedPv.Spec.CSI.VolumeAttributes[CsiVolumeMountAttributePvcName] - actualPvcNameAttr := actualPv.Spec.CSI.VolumeAttributes[CsiVolumeMountAttributePvcName] - if expectedPvcNameAttr[:len(expectedPvcNameAttr)-5] != actualPvcNameAttr[:len(actualPvcNameAttr)-5] { - return false - } - - if !utils.EqualStringMaps(getMountOptionsMap(expectedPv.Spec.MountOptions), getMountOptionsMap(actualPv.Spec.MountOptions)) { - return false - } - // ignore expectedPv.Spec.StorageClassName != actualPv.Spec.StorageClassName for transition period - if expectedPv.Spec.Capacity[corev1.ResourceStorage] != actualPv.Spec.Capacity[corev1.ResourceStorage] || - len(expectedPv.Spec.AccessModes) != len(actualPv.Spec.AccessModes) || - (len(expectedPv.Spec.AccessModes) != 1 && expectedPv.Spec.AccessModes[0] != actualPv.Spec.AccessModes[0]) || - expectedPv.Spec.CSI.Driver != actualPv.Spec.CSI.Driver { - return false - } - if expectedPv.Spec.CSI.NodeStageSecretRef != nil { - if actualPv.Spec.CSI.NodeStageSecretRef == nil || expectedPv.Spec.CSI.NodeStageSecretRef.Name != actualPv.Spec.CSI.NodeStageSecretRef.Name { - return false - } - } else if actualPv.Spec.CSI.NodeStageSecretRef != nil { - return false - } - if expectedPv.Spec.ClaimRef != nil { - if actualPv.Spec.ClaimRef == nil { // ignore || expectedPv.Spec.ClaimRef.Name != actualPv.Spec.ClaimRef.Name - return false - } - } else if actualPv.Spec.ClaimRef != nil { - return false - } - return true -} - func cloneMap(original map[string]string, ignoreKeys ...string) map[string]string { - copy := make(map[string]string, len(original)) + clonedMap := make(map[string]string, len(original)) ignoreKeysMap := convertToSet(ignoreKeys) for key, value := range original { if _, ok := ignoreKeysMap[key]; !ok { - copy[key] = value + clonedMap[key] = value } } - return copy + return clonedMap } func convertToSet(ignoreKeys []string) map[string]struct{} { @@ -142,17 +88,6 @@ func getPvAnnotations(pv *corev1.PersistentVolume) map[string]string { return annotations } -func getVolumeAttributes(pv *corev1.PersistentVolume) map[string]string { - attributes := make(map[string]string) - for key, value := range pv.Spec.CSI.VolumeAttributes { - if _, ok := ignoreVolumeAttributeKeys[key]; ok { - continue // ignore automatically added and name specific attribute(s) - } - attributes[key] = value - } - return attributes -} - func getMountOptionsMap(mountOptions []string) map[string]string { return slice.Reduce(mountOptions, make(map[string]string), func(acc map[string]string, item string) map[string]string { if len(item) == 0 { diff --git a/pkg/apis/internal/strings.go b/pkg/apis/internal/strings.go new file mode 100644 index 000000000..58a1fa1ed --- /dev/null +++ b/pkg/apis/internal/strings.go @@ -0,0 +1,9 @@ +package internal + +// EqualTillPostfix Compares two strings till the postfix +func EqualTillPostfix(value1, value2 string, postfixLength int) bool { + if len(value1) < postfixLength || len(value2) < postfixLength { + return false + } + return value1[:len(value1)-postfixLength] == value2[:len(value2)-postfixLength] +} diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index dcc3604ff..41a21a4e0 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -802,8 +802,8 @@ func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeCli if err != nil { return nil, err } - if pvcExist && - (!persistentvolume.EqualPersistentVolumeClaims(existingPvc, newPvc) || existingPvc.Spec.VolumeName != pvName) { + needToReCreatePvc := !persistentvolume.EqualPersistentVolumeClaims(existingPvc, newPvc) || existingPvc.Spec.VolumeName != pvName + if pvcExist && needToReCreatePvc { pvcName = newPvc.GetName() } if !existsPvByPvcName && !existsPvByVolumeContent { @@ -813,7 +813,7 @@ func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeCli return nil, err } } - if !pvcExist || !persistentvolume.EqualPersistentVolumeClaims(existingPvc, newPvc) { + if !pvcExist || needToReCreatePvc { newPvc.SetName(pvcName) log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for PersistentVolume %s", newPvc.GetName(), namespace, pvName) if _, err := kubeClient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, newPvc, metav1.CreateOptions{}); err != nil { diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index 74a294f77..d3644dbda 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "github.com/equinor/radix-operator/pkg/apis/defaults" + "github.com/equinor/radix-operator/pkg/apis/defaults/k8s" + "github.com/equinor/radix-operator/pkg/apis/internal" "github.com/google/uuid" "k8s.io/apimachinery/pkg/types" "strings" @@ -59,14 +61,14 @@ type volumeMountTestScenario struct { } type deploymentVolumesTestScenario struct { - name string - props expectedPvcPvProperties - radixVolumeMounts []v1.RadixVolumeMount - volumes []corev1.Volume - existingPVsBeforeTestRun []corev1.PersistentVolume - existingPvcsBeforeTestRun []corev1.PersistentVolumeClaim - existingPVsAfterTestRun []corev1.PersistentVolume - existingPvcsAfterTestRun []corev1.PersistentVolumeClaim + name string + props expectedPvcPvProperties + radixVolumeMounts []v1.RadixVolumeMount + volumes []corev1.Volume + existingPvs []corev1.PersistentVolume + existingPvcs []corev1.PersistentVolumeClaim + expectedPvs []corev1.PersistentVolume + expectedPvcs []corev1.PersistentVolumeClaim } type pvcTestScenario struct { @@ -493,8 +495,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { environment = "some-env" componentName = "some-component" ) - anotherNamespace := commonUtils.RandString(10) - anotherComponentName := commonUtils.RandString(10) + anotherNamespace := strings.ToLower(commonUtils.RandString(10)) + anotherComponentName := strings.ToLower(commonUtils.RandString(10)) var scenarios []deploymentVolumesTestScenario scenarios = append(scenarios, func() []deploymentVolumesTestScenario { @@ -508,12 +510,12 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), }, } @@ -546,25 +548,26 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { v.Name = scenarioProps.expectedVolumeName }), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + expectedPvcs: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName pvc.Spec.VolumeName = scenarioProps.expectedNewPvName }), }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + expectedPvs: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) { pv.ObjectMeta.Name = scenarioProps.expectedNewPvName pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + pv.Spec.ClaimRef.Name = scenarioProps.expectedNewPvcName pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName }), }, @@ -598,22 +601,22 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{ createRandomPvc(props, props.namespace, props.componentName), pvcForAnotherNamespace, pvcForAnotherComponent, }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + expectedPvcs: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), pvcForAnotherNamespace, pvcForAnotherComponent, }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{ createRandomPv(props, props.namespace, props.componentName), pvForAnotherNamespace, pvForAnotherComponent, }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + expectedPvs: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), pvForAnotherNamespace, pvForAnotherComponent, @@ -635,18 +638,18 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{ createRandomPvc(props, props.namespace, props.componentName), }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + expectedPvcs: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} }), }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{ createRandomPv(props, props.namespace, props.componentName), }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + expectedPvs: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) { pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") }), @@ -671,18 +674,18 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{ existingPvc, }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + expectedPvcs: []corev1.PersistentVolumeClaim{ modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} }), }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{ existingPv, }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + expectedPvs: []corev1.PersistentVolume{ modifyPv(existingPv, func(pv *corev1.PersistentVolume) { pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} pv.Spec.MountOptions = getMountOptions(props, false) @@ -708,18 +711,18 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{ existingPvc, }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + expectedPvcs: []corev1.PersistentVolumeClaim{ modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} }), }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{ existingPv, }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + expectedPvs: []corev1.PersistentVolume{ modifyPv(existingPv, func(pv *corev1.PersistentVolume) { pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} pv.Spec.MountOptions = getMountOptions(props, false) @@ -745,23 +748,23 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{ modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} }), }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + expectedPvcs: []corev1.PersistentVolumeClaim{ modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} }), }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{ modifyPv(existingPv, func(pv *corev1.PersistentVolume) { pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} pv.Spec.MountOptions = getMountOptions(props, false) }), }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + expectedPvs: []corev1.PersistentVolume{ modifyPv(existingPv, func(pv *corev1.PersistentVolume) { pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} pv.Spec.MountOptions = getMountOptions(props, true) @@ -784,12 +787,12 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) { pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") }), @@ -819,12 +822,12 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) { pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", @@ -863,12 +866,12 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{}, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{}, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) { pv.Spec.MountOptions = getMountOptions(props, true, "--use-adls=false") @@ -896,18 +899,18 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{ pvcForAnotherComponent, }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + expectedPvcs: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), pvcForAnotherComponent, }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{ existingPv, pvForAnotherComponent, }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + expectedPvs: []corev1.PersistentVolume{ existingPv, pvForAnotherComponent, }, @@ -932,18 +935,18 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), }, - existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ + existingPvcs: []corev1.PersistentVolumeClaim{ pvcForAnotherComponent, }, - existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ + expectedPvcs: []corev1.PersistentVolumeClaim{ createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), pvcForAnotherComponent, }, - existingPVsBeforeTestRun: []corev1.PersistentVolume{ + existingPvs: []corev1.PersistentVolume{ existingPv, pvForAnotherComponent, }, - existingPVsAfterTestRun: []corev1.PersistentVolume{ + expectedPvs: []corev1.PersistentVolume{ existingPv, pvForAnotherComponent, }, @@ -953,48 +956,48 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { getScenario(getPropsCsiBlobVolume1Storage1(nil)), } }()...) - // TODO fixing it - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) - // pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) - // expectedPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) - // matchPvAndPvc(&expectedPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Do not change existing PVC with class name, when creating new PersistentVolume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcsBeforeTestRun: []corev1.PersistentVolumeClaim{ - // pvcForAnotherComponent, - // existingPvc, - // }, - // existingPvcsAfterTestRun: []corev1.PersistentVolumeClaim{ - // pvcForAnotherComponent, - // existingPvc, - // }, - // existingPVsBeforeTestRun: []corev1.PersistentVolume{ - // pvForAnotherComponent, - // }, - // existingPVsAfterTestRun: []corev1.PersistentVolume{ - // pvForAnotherComponent, - // expectedPv, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) + pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) + expectedPvc := createRandomPvc(props, props.namespace, componentName) + expectedPv := createRandomPv(props, props.namespace, componentName) + matchPvAndPvc(&expectedPv, &expectedPvc) + return deploymentVolumesTestScenario{ + name: "Do not change existing PVC with class name, when creating new PersistentVolume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + expectedPvc, + }, + existingPvs: []corev1.PersistentVolume{ + pvForAnotherComponent, + }, + expectedPvs: []corev1.PersistentVolume{ + pvForAnotherComponent, + expectedPv, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { - for _, factory := range suite.radixCommonDeployComponentFactories { + for _, factory := range suite.radixCommonDeployComponentFactories[:1] { for _, scenario := range scenarios { t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) testEnv := getTestEnv() @@ -1009,15 +1012,22 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) require.NoError(t, err) - assert.True(t, equalPersistentVolumeClaims(&scenario.existingPvcsAfterTestRun, &existingPvcs), "PVC-s are not equal") - assert.True(t, equalPersistentVolumes(&scenario.existingPVsAfterTestRun, &existingPvs), "PV-s are not equal") + assert.True(t, equalPersistentVolumeClaims(&scenario.expectedPvcs, &existingPvcs), "PVC-s are not equal") + assert.True(t, equalPersistentVolumes(&scenario.expectedPvs, &existingPvs), "PV-s are not equal") } } }) } func matchPvAndPvc(pv *corev1.PersistentVolume, pvc *corev1.PersistentVolumeClaim) { - pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvc.Name + pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvc.GetName() + pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcNamespace] = pvc.GetNamespace() + pv.Spec.ClaimRef = &corev1.ObjectReference{ + APIVersion: "v1", + Kind: k8s.KindPersistentVolumeClaim, + Name: pvc.GetName(), + Namespace: pvc.GetNamespace(), + } pvc.Spec.VolumeName = pv.Name } @@ -1049,6 +1059,11 @@ func createRandomAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties return createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) { pvName := "pvc-" + uuid.NewString() pv.ObjectMeta.Name = pvName + if pv.ObjectMeta.Labels == nil { + pv.ObjectMeta.Labels = make(map[string]string) + } + pv.ObjectMeta.Labels[kube.RadixNamespace] = namespace + pv.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvName] = pvName }) } @@ -1124,10 +1139,10 @@ func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcPvProperties)) } func putExistingDeploymentVolumesScenarioDataToFakeCluster(kubeClient kubernetes.Interface, scenario *deploymentVolumesTestScenario) { - for _, pvc := range scenario.existingPvcsBeforeTestRun { + for _, pvc := range scenario.existingPvcs { _, _ = kubeClient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.Background(), &pvc, metav1.CreateOptions{}) } - for _, pv := range scenario.existingPVsBeforeTestRun { + for _, pv := range scenario.existingPvs { _, _ = kubeClient.CoreV1().PersistentVolumes().Create(context.Background(), &pv, metav1.CreateOptions{}) } } @@ -1245,10 +1260,10 @@ func createExpectedPv(props expectedPvcPvProperties, modify func(pv *corev1.Pers }, AccessModes: []corev1.PersistentVolumeAccessMode{props.volumeAccessMode}, ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", + Kind: k8s.KindPersistentVolumeClaim, Namespace: props.namespace, Name: props.pvcName, - APIVersion: "v1", - Kind: "PersistentVolumeClaim", }, StorageClassName: "", MountOptions: mountOptions, @@ -1298,10 +1313,10 @@ func createAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties, modi }, AccessModes: []corev1.PersistentVolumeAccessMode{props.volumeAccessMode}, ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", + Kind: k8s.KindPersistentVolumeClaim, Namespace: props.namespace, Name: props.pvcName, - APIVersion: "v1", - Kind: "PersistentVolumeClaim", }, StorageClassName: "some-storage-class", MountOptions: mountOptions, @@ -1498,7 +1513,7 @@ func equalPersistentVolumes(expectedPvs, actualPvs *[]corev1.PersistentVolume) b for _, expectedPv := range *expectedPvs { var hasEqualPv bool for _, actualPv := range *actualPvs { - if persistentvolume.EqualPersistentVolumesForTest(&expectedPv, &actualPv) { + if persistentvolume.EqualPersistentVolumes(&expectedPv, &actualPv) { hasEqualPv = true break } @@ -1510,14 +1525,14 @@ func equalPersistentVolumes(expectedPvs, actualPvs *[]corev1.PersistentVolume) b return true } -func equalPersistentVolumeClaims(pvcList1, pvcList2 *[]corev1.PersistentVolumeClaim) bool { - if len(*pvcList1) != len(*pvcList2) { +func equalPersistentVolumeClaims(list1, list2 *[]corev1.PersistentVolumeClaim) bool { + if len(*list1) != len(*list2) { return false } - for _, pvc1 := range *pvcList1 { + for _, pvc1 := range *list1 { var hasEqualPvc bool - for _, pvc2 := range *pvcList2 { - if equalTillPostfix(pvc1.GetName(), pvc2.GetName(), 5) && + for _, pvc2 := range *list2 { + if internal.EqualTillPostfix(pvc1.GetName(), pvc2.GetName(), 5) && equalPrefix(pvc1.Spec.VolumeName, pvc2.Spec.VolumeName, 20) && persistentvolume.EqualPersistentVolumeClaims(&pvc1, &pvc2) { hasEqualPvc = true @@ -1535,14 +1550,5 @@ func equalPrefix(value1, value2 string, prefixLength int) bool { if len(value1) < prefixLength || len(value2) < prefixLength { return false } - eq := value1[:prefixLength] == value2[:prefixLength] - return eq -} - -func equalTillPostfix(value1, value2 string, postfixLength int) bool { - if len(value1) < postfixLength || len(value2) < postfixLength { - return false - } - eq := value1[:len(value1)-postfixLength] == value2[:len(value2)-postfixLength] - return eq + return value1[:prefixLength] == value2[:prefixLength] } From ac69182e32b276ae9727339cbb3a9db0a482f6e6 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Mon, 13 Jan 2025 17:20:34 +0100 Subject: [PATCH 60/68] Correcting pv-pvc re-creation logic --- pkg/apis/batch/syncer.go | 2 +- pkg/apis/deployment/deployment.go | 7 +- pkg/apis/deployment/kubedeployment.go | 4 +- pkg/apis/volumemount/volumemount.go | 309 ++++--- pkg/apis/volumemount/volumemount_test.go | 1029 +++++++++++----------- 5 files changed, 735 insertions(+), 616 deletions(-) diff --git a/pkg/apis/batch/syncer.go b/pkg/apis/batch/syncer.go index 7eb1d3571..b0ad97382 100644 --- a/pkg/apis/batch/syncer.go +++ b/pkg/apis/batch/syncer.go @@ -105,7 +105,7 @@ func (s *syncer) reconcile(ctx context.Context) error { if err != nil { return err } - actualVolumes, err := volumemount.CreateOrUpdateCsiAzureVolumeResources(ctx, s.kubeUtil.KubeClient(), rd, namespace, jobComponent, desiredVolumes) + actualVolumes, err := volumemount.CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent(ctx, s.kubeUtil.KubeClient(), rd, namespace, jobComponent, desiredVolumes) if err != nil { return fmt.Errorf("failed to create or update csi azure volume resources: %w", err) } diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 620df1918..600d49160 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/equinor/radix-operator/pkg/apis/volumemount" "sort" "strings" "time" @@ -531,7 +532,7 @@ func (deploy *Deployment) syncDeploymentForRadixComponent(ctx context.Context, c return fmt.Errorf("failed to create service account: %w", err) } - err = deploy.reconcileDeployment(ctx, component) + err = deploy.reconcileDeployComponent(ctx, component) if err != nil { return fmt.Errorf("failed to create deployment: %w", err) } @@ -585,6 +586,10 @@ func (deploy *Deployment) syncDeploymentForRadixComponent(ctx context.Context, c } } + if err = volumemount.GarbageCollectCsiAzureVolumeResourcesForDeployComponent(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace()); err != nil { + return fmt.Errorf("failed to garbage collect persistent volumes or persistent volume claims: %w", err) + } + return nil } diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index e36815b25..fe76854d7 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployComponent v1.RadixCommonDeployComponent) error { +func (deploy *Deployment) reconcileDeployComponent(ctx context.Context, deployComponent v1.RadixCommonDeployComponent) error { namespace := deploy.radixDeployment.Namespace currentDeployment, desiredDeployment, err := deploy.getCurrentAndDesiredDeployment(ctx, namespace, deployComponent) if err != nil { @@ -39,7 +39,7 @@ func (deploy *Deployment) reconcileDeployment(ctx context.Context, deployCompone return err } } - actualVolumes, err := volumemount.CreateOrUpdateCsiAzureVolumeResources(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace(), deployComponent, desiredDeployment.Spec.Template.Spec.Volumes) + actualVolumes, err := volumemount.CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent(ctx, deploy.kubeutil.KubeClient(), deploy.radixDeployment, deploy.radixDeployment.GetNamespace(), deployComponent, desiredDeployment.Spec.Template.Spec.Volumes) if err != nil { return err } diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index 41a21a4e0..20ba9a3da 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -40,8 +40,8 @@ const ( ) var ( - csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}} - functionalPersistentVolumePhases = []corev1.PersistentVolumePhase{corev1.VolumePending, corev1.VolumeBound, corev1.VolumeAvailable} + csiVolumeProvisioners = map[string]any{provisionerBlobCsiAzure: struct{}{}} + functionalPvPhases = []corev1.PersistentVolumePhase{corev1.VolumePending, corev1.VolumeBound, corev1.VolumeAvailable} ) // GetRadixDeployComponentVolumeMounts Gets list of v1.VolumeMount for radixv1.RadixCommonDeployComponent @@ -94,22 +94,37 @@ func GarbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx context.Con return garbageCollectSecrets(ctx, kubeUtil, namespace, secrets, excludeSecretNames) } -// CreateOrUpdateCsiAzureVolumeResources Create or update CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims, PersistentVolume +// CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent Create or update CSI Azure volume resources for a DeployComponent - PersistentVolumes, PersistentVolumeClaims, PersistentVolume // Returns actual volumes, with existing relevant PersistentVolumeClaimName and PersistentVolumeName -func CreateOrUpdateCsiAzureVolumeResources(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, deployComponent radixv1.RadixCommonDeployComponent, desiredVolumes []corev1.Volume) ([]corev1.Volume, error) { +func CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, deployComponent radixv1.RadixCommonDeployComponent, desiredVolumes []corev1.Volume) ([]corev1.Volume, error) { componentName := deployComponent.GetName() actualVolumes, err := createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx, kubeClient, radixDeployment, namespace, componentName, deployComponent.GetIdentity(), desiredVolumes) if err != nil { return nil, err } - currentlyUsedPvcNames, err := getCurrentlyUsedPersistentVolumeClaims(ctx, kubeClient, radixDeployment, actualVolumes) + return actualVolumes, nil +} + +// GarbageCollectCsiAzureVolumeResourcesForDeployComponent Garbage collect CSI Azure volume resources - PersistentVolumes, PersistentVolumeClaims +func GarbageCollectCsiAzureVolumeResourcesForDeployComponent(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string) error { + currentlyUsedPvcNames, err := getCurrentlyUsedPvcNames(ctx, kubeClient, radixDeployment) if err != nil { - return nil, err + return err } - if err = garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx, kubeClient, namespace, componentName, currentlyUsedPvcNames); err != nil { - return nil, err + var errs []error + if err = garbageCollectCsiAzurePvcs(ctx, kubeClient, namespace, currentlyUsedPvcNames); err != nil { + errs = append(errs, err) + } + if err = garbageCollectCsiAzurePvs(ctx, kubeClient, namespace, currentlyUsedPvcNames); err != nil { + errs = append(errs, err) + } + if err = garbageCollectOrphanedCsiAzurePvs(ctx, kubeClient, currentlyUsedPvcNames); err != nil { + errs = append(errs, err) } - return actualVolumes, garbageCollectOrphanedCsiAzurePersistentVolumes(ctx, kubeClient, currentlyUsedPvcNames) + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil } // CreateOrUpdateVolumeMountSecrets creates or updates secrets for volume mounts @@ -139,7 +154,7 @@ func GetCsiAzureVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) radi } } -func getCsiPersistentVolumesForNamespace(ctx context.Context, kubeClient kubernetes.Interface, namespace string, onlyFunctional bool) ([]corev1.PersistentVolume, error) { +func getCsiAzurePvsForNamespace(ctx context.Context, kubeClient kubernetes.Interface, namespace string, onlyFunctional bool) ([]corev1.PersistentVolume, error) { pvList, err := kubeClient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) if err != nil { return nil, err @@ -350,7 +365,7 @@ func getVolumeSource(componentName string, volumeMount *radixv1.RadixVolumeMount } func getCsiAzureVolumeSource(componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { - pvcName, err := getCsiAzurePersistentVolumeClaimName(componentName, radixVolumeMount) + pvcName, err := getCsiAzurePvcName(componentName, radixVolumeMount) if err != nil { return nil, err } @@ -395,7 +410,7 @@ func getVolumeMountDeprecatedVolumeName(volumeMount *radixv1.RadixVolumeMount, c return "", fmt.Errorf("unsupported volume type %s", volumeMount.Type) } -func getCsiAzurePersistentVolumeClaimName(componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { +func getCsiAzurePvcName(componentName string, radixVolumeMount *radixv1.RadixVolumeMount) (string, error) { volumeName, err := getCsiAzureVolumeMountName(componentName, radixVolumeMount) if err != nil { return "", err @@ -403,7 +418,7 @@ func getCsiAzurePersistentVolumeClaimName(componentName string, radixVolumeMount return fmt.Sprintf(persistentvolume.CsiPersistentVolumeClaimNameTemplate, volumeName, strings.ToLower(commonUtils.RandString(5))), nil } -func getCsiAzurePersistentVolumeName() string { +func getCsiAzurePvName() string { return fmt.Sprintf(persistentvolume.CsiPersistentVolumeNameTemplate, uuid.New().String()) } @@ -445,24 +460,19 @@ func pvIsForNamespace(pv corev1.PersistentVolume, namespace string) bool { } func pvIsFunctional(pv corev1.PersistentVolume) bool { - return slice.Any(functionalPersistentVolumePhases, func(phase corev1.PersistentVolumePhase) bool { return pv.Status.Phase == phase }) + // not Terminating or Released + return slice.Any(functionalPvPhases, func(phase corev1.PersistentVolumePhase) bool { return pv.Status.Phase == phase }) } -func getCsiAzurePersistentVolumeClaims(ctx context.Context, kubeClient kubernetes.Interface, namespace, componentName string) (*corev1.PersistentVolumeClaimList, error) { - return kubeClient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: getLabelSelectorForCsiAzurePersistenceVolumeClaim(componentName), - }) +func getCsiAzurePvcs(ctx context.Context, kubeClient kubernetes.Interface, namespace string) (*corev1.PersistentVolumeClaimList, error) { + return kubeClient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) } -func getLabelSelectorForCsiAzurePersistenceVolumeClaim(componentName string) string { +func getLabelSelectorForCsiAzurePvc(componentName string) string { return fmt.Sprintf("%s=%s, %s in (%s, %s)", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure)) } -func buildPersistentVolumeClaim(appName, namespace, componentName, pvName string, radixVolumeMount *radixv1.RadixVolumeMount) (*corev1.PersistentVolumeClaim, error) { - pvcName, err := getCsiAzurePersistentVolumeClaimName(componentName, radixVolumeMount) - if err != nil { - return nil, err - } +func buildPvc(appName, namespace, componentName, pvName, pvcName string, radixVolumeMount *radixv1.RadixVolumeMount) *corev1.PersistentVolumeClaim { return &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, @@ -483,7 +493,7 @@ func buildPersistentVolumeClaim(appName, namespace, componentName, pvName string StorageClassName: pointers.Ptr(""), // use "" to avoid to use the "default" storage class VolumeMode: pointers.Ptr(corev1.PersistentVolumeFilesystem), }, - }, nil + } } func getVolumeCapacity(radixVolumeMount *radixv1.RadixVolumeMount) resource.Quantity { @@ -494,33 +504,41 @@ func getVolumeCapacity(radixVolumeMount *radixv1.RadixVolumeMount) resource.Quan return requestsVolumeMountSize } -func populateCsiAzurePersistentVolume(persistentVolume *corev1.PersistentVolume, appName, namespace, componentName, pvName, pvcName string, radixVolumeMount *radixv1.RadixVolumeMount, identity *radixv1.Identity) *corev1.PersistentVolume { +func buildCsiAzurePv(appName, namespace, componentName, pvName, pvcName string, radixVolumeMount *radixv1.RadixVolumeMount, identity *radixv1.Identity) *corev1.PersistentVolume { identityClientId := getIdentityClientId(identity) useAzureIdentity := getUseAzureIdentity(identity, radixVolumeMount.UseAzureIdentity) csiVolumeCredSecretName := defaults.GetCsiAzureVolumeMountCredsSecretName(componentName, radixVolumeMount.Name) - persistentVolume.ObjectMeta.Name = pvName - persistentVolume.ObjectMeta.Labels = getCsiAzurePersistentVolumeLabels(appName, namespace, componentName, radixVolumeMount) - persistentVolume.ObjectMeta.Annotations = getCsiAzurePersistentVolumeAnnotations(csiVolumeCredSecretName, namespace, useAzureIdentity) - persistentVolume.Spec.StorageClassName = "" - persistentVolume.Spec.MountOptions = getCsiAzurePersistentVolumeMountOptionsForAzureBlob(radixVolumeMount) - persistentVolume.Spec.Capacity = corev1.ResourceList{corev1.ResourceStorage: getVolumeCapacity(radixVolumeMount)} - persistentVolume.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{getVolumeMountAccessMode(radixVolumeMount)} - persistentVolume.Spec.ClaimRef = &corev1.ObjectReference{ - APIVersion: "v1", - Kind: k8s.KindPersistentVolumeClaim, - Namespace: namespace, - Name: pvcName, - } - persistentVolume.Spec.CSI = &corev1.CSIPersistentVolumeSource{ - Driver: provisionerBlobCsiAzure, - VolumeHandle: getVolumeHandle(namespace, componentName, pvName, getRadixVolumeMountStorage(radixVolumeMount)), - VolumeAttributes: getCsiAzurePersistentVolumeAttributes(namespace, radixVolumeMount, pvName, pvcName, useAzureIdentity, identityClientId), + pv := corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, + Labels: getCsiAzurePvLabels(appName, namespace, componentName, radixVolumeMount), + Annotations: getCsiAzurePvAnnotations(csiVolumeCredSecretName, namespace, useAzureIdentity), + }, + Spec: corev1.PersistentVolumeSpec{ + StorageClassName: "", + MountOptions: getCsiAzurePvMountOptionsForAzureBlob(radixVolumeMount), + Capacity: corev1.ResourceList{corev1.ResourceStorage: getVolumeCapacity(radixVolumeMount)}, + AccessModes: []corev1.PersistentVolumeAccessMode{getVolumeMountAccessMode(radixVolumeMount)}, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", + Kind: k8s.KindPersistentVolumeClaim, + Namespace: namespace, + Name: pvcName, + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: provisionerBlobCsiAzure, + VolumeHandle: getVolumeHandle(namespace, componentName, pvName, getRadixVolumeMountStorage(radixVolumeMount)), + VolumeAttributes: getCsiAzurePvAttributes(namespace, radixVolumeMount, pvName, pvcName, useAzureIdentity, identityClientId), + }, + }, + }, } if !useAzureIdentity { - persistentVolume.Spec.CSI.NodeStageSecretRef = &corev1.SecretReference{Name: csiVolumeCredSecretName, Namespace: namespace} + pv.Spec.CSI.NodeStageSecretRef = &corev1.SecretReference{Name: csiVolumeCredSecretName, Namespace: namespace} } - persistentVolume.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimRetain. PersistentVolumeReclaimPolicy deletes volume on unmount. - return persistentVolume + pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain // Using only PersistentVolumeReclaimRetain. PersistentVolumeReclaimPolicy deletes volume on unmount. + return &pv } func getVolumeHandle(namespace, componentName, pvName, storageName string) string { @@ -540,7 +558,7 @@ func getIdentityClientId(identity *radixv1.Identity) string { return "" } -func getCsiAzurePersistentVolumeAnnotations(csiVolumeCredSecretName, namespace string, useAzureIdentity bool) map[string]string { +func getCsiAzurePvAnnotations(csiVolumeCredSecretName, namespace string, useAzureIdentity bool) map[string]string { annotationsMap := map[string]string{ persistentvolume.CsiAnnotationProvisionedBy: provisionerBlobCsiAzure, } @@ -551,7 +569,7 @@ func getCsiAzurePersistentVolumeAnnotations(csiVolumeCredSecretName, namespace s return annotationsMap } -func getCsiAzurePersistentVolumeLabels(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { +func getCsiAzurePvLabels(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) map[string]string { return map[string]string{ kube.RadixAppLabel: appName, kube.RadixNamespace: namespace, @@ -560,7 +578,7 @@ func getCsiAzurePersistentVolumeLabels(appName, namespace, componentName string, } } -func getCsiAzurePersistentVolumeAttributes(namespace string, radixVolumeMount *radixv1.RadixVolumeMount, pvName, pvcName string, useAzureIdentity bool, clientId string) map[string]string { +func getCsiAzurePvAttributes(namespace string, radixVolumeMount *radixv1.RadixVolumeMount, pvName, pvcName string, useAzureIdentity bool, clientId string) map[string]string { attributes := make(map[string]string) switch GetCsiAzureVolumeMountType(radixVolumeMount) { case radixv1.MountTypeBlobFuse2FuseCsiAzure: @@ -589,7 +607,7 @@ func getCsiAzurePersistentVolumeAttributes(namespace string, radixVolumeMount *r return attributes } -func getCsiAzurePersistentVolumeMountOptionsForAzureBlob(radixVolumeMount *radixv1.RadixVolumeMount) []string { +func getCsiAzurePvMountOptionsForAzureBlob(radixVolumeMount *radixv1.RadixVolumeMount) []string { mountOptions := []string{ "--file-cache-timeout-in-seconds=120", "--use-attr-cache=true", @@ -689,18 +707,18 @@ func getRadixBlobFuse2VolumeMountRequestsStorage(radixVolumeMount *radixv1.Radix return radixVolumeMount.RequestsStorage } -func deletePersistentVolumeClaim(ctx context.Context, kubeClient kubernetes.Interface, namespace, pvcName string) error { +func deletePvc(ctx context.Context, kubeClient kubernetes.Interface, namespace, pvcName string) error { if len(namespace) == 0 || len(pvcName) == 0 { log.Ctx(ctx).Debug().Msgf("Skip deleting PVC - namespace %s or name %s is empty", namespace, pvcName) return nil } - if err := kubeClient.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil { // && !k8serrors.IsNotFound(err) { + if err := kubeClient.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { return err } return nil } -func deletePersistentVolume(ctx context.Context, kubeClient kubernetes.Interface, pvName string) error { +func deletePv(ctx context.Context, kubeClient kubernetes.Interface, pvName string) error { if len(pvName) == 0 { log.Ctx(ctx).Debug().Msg("Skip deleting PersistentVolume - name is empty") return nil @@ -719,7 +737,7 @@ func getRadixVolumeMountStorage(radixVolumeMount *radixv1.RadixVolumeMount) stri return radixVolumeMount.Storage } -func garbageCollectOrphanedCsiAzurePersistentVolumes(ctx context.Context, kubeClient kubernetes.Interface, excludePvcNames map[string]any) error { +func garbageCollectOrphanedCsiAzurePvs(ctx context.Context, kubeClient kubernetes.Interface, excludePvcNames map[string]any) error { pvList, err := kubeClient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) if err != nil { return err @@ -736,7 +754,7 @@ func garbageCollectOrphanedCsiAzurePersistentVolumes(ctx context.Context, kubeCl continue } log.Ctx(ctx).Info().Msgf("Delete orphaned Csi Azure PersistantVolume %s of PersistantVolumeClaim %s", pv.Name, pv.Spec.ClaimRef.Name) - if err := deletePersistentVolume(ctx, kubeClient, pv.Name); err != nil && !k8serrors.IsNotFound(err) { + if err := deletePv(ctx, kubeClient, pv.Name); err != nil && !k8serrors.IsNotFound(err) { errs = append(errs, err) } } @@ -752,11 +770,11 @@ func knownCSIDriver(driver string) bool { } func createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace, componentName string, identity *radixv1.Identity, desiredVolumes []corev1.Volume) ([]corev1.Volume, error) { - functionalPvList, err := getCsiPersistentVolumesForNamespace(ctx, kubeClient, namespace, true) + functionalPvList, err := getCsiAzurePvsForNamespace(ctx, kubeClient, namespace, true) if err != nil { return nil, err } - pvcByNameMap, err := getPvcByNameMap(ctx, kubeClient, namespace, componentName) + pvcByNameMap, err := getComponentPvcByNameMap(ctx, kubeClient, namespace, componentName) if err != nil { return nil, err } @@ -787,36 +805,68 @@ func createOrUpdateCsiAzureVolumeResourcesForVolumes(ctx context.Context, kubeCl return volumes, nil } -func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, componentName string, identity *radixv1.Identity, volume corev1.Volume, radixVolumeMount *radixv1.RadixVolumeMount, functionalPvList []corev1.PersistentVolume, pvcByNameMap map[string]*corev1.PersistentVolumeClaim) (*corev1.Volume, error) { +func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, namespace string, componentName string, identity *radixv1.Identity, volume corev1.Volume, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumes []corev1.PersistentVolume, pvcByNameMap map[string]*corev1.PersistentVolumeClaim) (*corev1.Volume, error) { if volume.PersistentVolumeClaim == nil { return &volume, nil } appName := radixDeployment.Spec.AppName pvcName := volume.PersistentVolumeClaim.ClaimName - pvName, existsPvByPvcName, existsPvByVolumeContent := getActualExistingCsiAzurePvName(appName, namespace, componentName, radixVolumeMount, functionalPvList, pvcName, identity) - if !existsPvByPvcName && !existsPvByVolumeContent { - pvName = getCsiAzurePersistentVolumeName() + existingPvc := pvcByNameMap[pvcName] + + pvName := getCsiAzurePvName() + existingPv, pvExists := getCsiAzureComponentPvByPvcName(persistentVolumes, pvcName) + if pvExists { + radixVolumeMountPv := buildCsiAzurePv(appName, namespace, componentName, existingPv.GetName(), pvcName, radixVolumeMount, identity) + if !matchComponentVolumeMountLabels(namespace, appName, componentName, radixVolumeMount, existingPv, existingPvc) || + !persistentvolume.EqualPersistentVolumes(existingPv, radixVolumeMountPv) { + pvExists = false + newPvcName, err := getCsiAzurePvcName(componentName, radixVolumeMount) + if err != nil { + return nil, err + } + pvcName = newPvcName + } + } else { + existingPv, pvExists = getCsiAzureComponentPvByRadixVolumeMountContent(namespace, appName, componentName, radixVolumeMount, persistentVolumes, pvcName, existingPvc, identity) } - existingPvc, pvcExist := pvcByNameMap[pvcName] - newPvc, err := buildPersistentVolumeClaim(appName, namespace, componentName, pvName, radixVolumeMount) - if err != nil { - return nil, err + if pvExists { + pvName = existingPv.GetName() + pvcName = existingPv.Spec.ClaimRef.Name + } + + existingPvc, pvcExists := pvcByNameMap[pvcName] + if !pvExists && pvcExists { + pvName = existingPvc.Spec.VolumeName } - needToReCreatePvc := !persistentvolume.EqualPersistentVolumeClaims(existingPvc, newPvc) || existingPvc.Spec.VolumeName != pvName - if pvcExist && needToReCreatePvc { - pvcName = newPvc.GetName() + needToCreatePvc := !pvcExists + needToReCreatePv := pvExists && !pvcExists && len(existingPv.Spec.StorageClassName) > 0 // HACK: always re-create PV if it uses SC and PVC is missing + radixVolumeMountPvc := buildPvc(appName, namespace, componentName, pvName, pvcName, radixVolumeMount) + needToReCreatePvc := pvcExists && (existingPvc.Spec.VolumeName != pvName || !persistentvolume.EqualPersistentVolumeClaims(existingPvc, radixVolumeMountPvc)) + + if needToReCreatePv || needToReCreatePvc { + newPvcName, err := getCsiAzurePvcName(componentName, radixVolumeMount) + if err != nil { + return nil, err + } + pvcName = newPvcName + + pvExists = false + pvName = getCsiAzurePvName() } - if !existsPvByPvcName && !existsPvByVolumeContent { + + if !pvExists || needToReCreatePv { log.Ctx(ctx).Debug().Msgf("Create PersistentVolume %s in namespace %s", pvName, namespace) - pv := populateCsiAzurePersistentVolume(&corev1.PersistentVolume{}, appName, namespace, componentName, pvName, pvcName, radixVolumeMount, identity) - if _, err = kubeClient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil { + pv := buildCsiAzurePv(appName, namespace, componentName, pvName, pvcName, radixVolumeMount, identity) + if _, err := kubeClient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil { return nil, err } } - if !pvcExist || needToReCreatePvc { - newPvc.SetName(pvcName) - log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for PersistentVolume %s", newPvc.GetName(), namespace, pvName) - if _, err := kubeClient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, newPvc, metav1.CreateOptions{}); err != nil { + + if needToCreatePvc || needToReCreatePvc { + radixVolumeMountPvc.SetName(pvcName) + radixVolumeMountPvc.Spec.VolumeName = pvName + log.Ctx(ctx).Debug().Msgf("Create PersistentVolumeClaim %s in namespace %s for PersistentVolume %s", radixVolumeMountPvc.GetName(), namespace, pvName) + if _, err := kubeClient.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, radixVolumeMountPvc, metav1.CreateOptions{}); err != nil { return nil, err } } @@ -824,36 +874,38 @@ func createOrUpdateCsiAzureVolumeResourcesForVolume(ctx context.Context, kubeCli return &volume, nil } -func getPvcByNameMap(ctx context.Context, kubeClient kubernetes.Interface, namespace string, componentName string) (map[string]*corev1.PersistentVolumeClaim, error) { - pvcList, err := getCsiAzurePersistentVolumeClaims(ctx, kubeClient, namespace, componentName) +func getComponentPvcByNameMap(ctx context.Context, kubeClient kubernetes.Interface, namespace string, componentName string) (map[string]*corev1.PersistentVolumeClaim, error) { + pvcList, err := getCsiAzurePvcs(ctx, kubeClient, namespace) if err != nil { return nil, err } - return persistentvolume.GetPersistentVolumeClaimMap(&pvcList.Items), nil + pvcs := slice.FindAll(pvcList.Items, func(pvc corev1.PersistentVolumeClaim) bool { + return pvc.GetLabels()[kube.RadixComponentLabel] == componentName + }) + return persistentvolume.GetPersistentVolumeClaimMap(&pvcs), nil } -func getCurrentlyUsedPersistentVolumeClaims(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment, volumes []corev1.Volume) (map[string]any, error) { +func getCurrentlyUsedPvcNames(ctx context.Context, kubeClient kubernetes.Interface, radixDeployment *radixv1.RadixDeployment) (map[string]any, error) { namespace := radixDeployment.GetNamespace() - pvcNames := make(map[string]any) - deploymentList, err := kubeClient.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) + currentlyUsedPvcNames := make(map[string]any) + existingDeployments, err := kubeClient.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) if err != nil { return nil, err } - for _, deployment := range deploymentList.Items { - pvcNames = appendUsedPersistenceVolumeClaimsFrom(pvcNames, deployment.Spec.Template.Spec.Volumes) + for _, deployment := range existingDeployments.Items { + currentlyUsedPvcNames = appendPvcNamesFromVolumes(currentlyUsedPvcNames, deployment.Spec.Template.Spec.Volumes) } - jobsList, err := kubeClient.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{}) + existingJobs, err := kubeClient.BatchV1().Jobs(namespace).List(ctx, metav1.ListOptions{}) if err != nil { return nil, err } - for _, job := range jobsList.Items { - pvcNames = appendUsedPersistenceVolumeClaimsFrom(pvcNames, job.Spec.Template.Spec.Volumes) + for _, job := range existingJobs.Items { + currentlyUsedPvcNames = appendPvcNamesFromVolumes(currentlyUsedPvcNames, job.Spec.Template.Spec.Volumes) } - // TODO add from RadixDeployments, connected to existing scheduled jobs and batches - return appendUsedPersistenceVolumeClaimsFrom(pvcNames, volumes), nil + return currentlyUsedPvcNames, nil } -func appendUsedPersistenceVolumeClaimsFrom(pvcMap map[string]any, volumes []corev1.Volume) map[string]any { +func appendPvcNamesFromVolumes(pvcMap map[string]any, volumes []corev1.Volume) map[string]any { return slice.Reduce(volumes, pvcMap, func(acc map[string]any, volume corev1.Volume) map[string]any { if volume.PersistentVolumeClaim != nil && len(volume.PersistentVolumeClaim.ClaimName) > 0 { acc[volume.PersistentVolumeClaim.ClaimName] = struct{}{} @@ -862,8 +914,24 @@ func appendUsedPersistenceVolumeClaimsFrom(pvcMap map[string]any, volumes []core }) } -func garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx context.Context, kubeClient kubernetes.Interface, namespace, componentName string, excludePvcNames map[string]any) error { - pvcList, err := getCsiAzurePersistentVolumeClaims(ctx, kubeClient, namespace, componentName) +func garbageCollectCsiAzurePvs(ctx context.Context, kubeClient kubernetes.Interface, namespace string, excludePvcNames map[string]any) error { + pvs, err := getCsiAzurePvsForNamespace(ctx, kubeClient, namespace, true) + if err != nil { + return err + } + for _, pv := range pvs { + if _, ok := excludePvcNames[pv.Spec.ClaimRef.Name]; ok { + continue + } + log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolume %s in namespace %s", pv.Name, namespace) + if err = deletePv(ctx, kubeClient, pv.Name); err != nil { + return err + } + } + return nil +} +func garbageCollectCsiAzurePvcs(ctx context.Context, kubeClient kubernetes.Interface, namespace string, excludePvcNames map[string]any) error { + pvcList, err := getCsiAzurePvcs(ctx, kubeClient, namespace) if err != nil { return err } @@ -873,13 +941,13 @@ func garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx contex continue } log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolumeClaim %s in namespace %s", pvc.Name, namespace) - if err := deletePersistentVolumeClaim(ctx, kubeClient, namespace, pvc.Name); err != nil { + if err := deletePvc(ctx, kubeClient, namespace, pvc.Name); err != nil { errs = append(errs, err) continue } pvName := pvc.Spec.VolumeName log.Ctx(ctx).Debug().Msgf("Delete not used CSI Azure PersistentVolume %s in namespace %s", pvName, namespace) - if err := deletePersistentVolume(ctx, kubeClient, pvName); err != nil { + if err := deletePv(ctx, kubeClient, pvName); err != nil { errs = append(errs, err) } } @@ -889,39 +957,48 @@ func garbageCollectCsiAzurePersistentVolumeClaimsAndPersistentVolumes(ctx contex return nil } -func getActualExistingCsiAzurePvName(appName, namespace, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumes []corev1.PersistentVolume, pvcName string, identity *radixv1.Identity) (string, bool, bool) { - if pvName, ok := getActualCsiAzurePvNameByPvcName(appName, namespace, componentName, radixVolumeMount, persistentVolumes, pvcName, identity); ok { - return pvName, true, true - } - if pvName, ok := getFirstActualCsiAzurePvName(appName, namespace, componentName, radixVolumeMount, persistentVolumes, pvcName, identity); ok { - return pvName, false, true - } - return "", false, false +func getCsiAzureComponentPvByPvcName(persistentVolumes []corev1.PersistentVolume, pvcName string) (*corev1.PersistentVolume, bool) { + pv, ok := slice.FindFirst(persistentVolumes, func(pv corev1.PersistentVolume) bool { + return pv.Spec.PersistentVolumeSource.CSI != nil && pv.Spec.ClaimRef != nil && pv.Spec.ClaimRef.Name == pvcName + }) + return &pv, ok } -func getFirstActualCsiAzurePvName(appName string, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumes []corev1.PersistentVolume, pvcName string, identity *radixv1.Identity) (string, bool) { +func getCsiAzureComponentPvByRadixVolumeMountContent(appName string, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumes []corev1.PersistentVolume, pvcName string, pvc *corev1.PersistentVolumeClaim, identity *radixv1.Identity) (*corev1.PersistentVolume, bool) { for _, pv := range persistentVolumes { - if pv.Spec.ClaimRef == nil { + if pv.Spec.PersistentVolumeSource.CSI == nil || + pv.Spec.ClaimRef == nil || + !matchComponentVolumeMountLabels(namespace, appName, componentName, radixVolumeMount, &pv, pvc) { continue } - desiredPv := populateCsiAzurePersistentVolume(pv.DeepCopy(), appName, namespace, componentName, pv.GetName(), pvcName, radixVolumeMount, identity) - if persistentvolume.EqualPersistentVolumes(&pv, desiredPv) { - return pv.GetName(), true + radixVolumeMountPv := buildCsiAzurePv(appName, namespace, componentName, pv.GetName(), pvcName, radixVolumeMount, identity) + if persistentvolume.EqualPersistentVolumes(&pv, radixVolumeMountPv) { + return &pv, true } } - return "", false + return nil, false } -func getActualCsiAzurePvNameByPvcName(appName string, namespace string, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, persistentVolumes []corev1.PersistentVolume, pvcName string, identity *radixv1.Identity) (string, bool) { - if pv, ok := slice.FindFirst(persistentVolumes, func(pv corev1.PersistentVolume) bool { - return pv.Spec.ClaimRef != nil && pv.Spec.ClaimRef.Name == pvcName - }); ok { - desiredPv := populateCsiAzurePersistentVolume(pv.DeepCopy(), appName, namespace, componentName, pv.GetName(), pvcName, radixVolumeMount, identity) - if persistentvolume.EqualPersistentVolumes(&pv, desiredPv) { - return pv.GetName(), true - } +func matchComponentVolumeMountLabels(namespace, appName, componentName string, radixVolumeMount *radixv1.RadixVolumeMount, pv *corev1.PersistentVolume, pvc *corev1.PersistentVolumeClaim) bool { + pvLabels := pv.GetLabels() + if _, ok := pvLabels[kube.RadixAppLabel]; ok && matchComponentVolumeMountPvLabels(pvLabels, namespace, appName, componentName, radixVolumeMount) { + return true } - return "", false + return pvc != nil && matchComponentVolumeMountPvcLabels(pvc.GetLabels(), appName, componentName, radixVolumeMount) +} + +func matchComponentVolumeMountPvLabels(labels map[string]string, namespace, appName, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) bool { + return labels[kube.RadixNamespace] == namespace && + labels[kube.RadixAppLabel] == appName && + labels[kube.RadixComponentLabel] == componentName && + labels[kube.RadixVolumeMountNameLabel] == radixVolumeMount.Name +} + +func matchComponentVolumeMountPvcLabels(labels map[string]string, appName, componentName string, radixVolumeMount *radixv1.RadixVolumeMount) bool { + return labels[kube.RadixAppLabel] == appName && + labels[kube.RadixComponentLabel] == componentName && + labels[kube.RadixVolumeMountNameLabel] == radixVolumeMount.Name && + labels[kube.RadixMountTypeLabel] == string(GetCsiAzureVolumeMountType(radixVolumeMount)) } func getRadixVolumeMountsByNameMap(radixDeployment *radixv1.RadixDeployment, componentName string) map[string]*radixv1.RadixVolumeMount { diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index d3644dbda..042fca4f6 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -495,451 +495,423 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { environment = "some-env" componentName = "some-component" ) - anotherNamespace := strings.ToLower(commonUtils.RandString(10)) + //anotherNamespace := strings.ToLower(commonUtils.RandString(10)) anotherComponentName := strings.ToLower(commonUtils.RandString(10)) + anotherVolumeMountName := strings.ToLower(commonUtils.RandString(10)) var scenarios []deploymentVolumesTestScenario - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcs: []corev1.PersistentVolumeClaim{}, - expectedPvcs: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPvs: []corev1.PersistentVolume{}, - expectedPvs: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - type scenarioProperties struct { - changedNewRadixVolumeName string - changedNewRadixVolumeStorageName string - expectedVolumeName string - expectedNewSecretName string - expectedNewPvcName string - expectedNewPvName string - } - getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Update storage in existing volume name and storage", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.Name = scenarioProps.changedNewRadixVolumeName - vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) { - v.Name = scenarioProps.expectedVolumeName - }), - }, - existingPvcs: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - expectedPvcs: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - }), - }, - existingPvs: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - }, - expectedPvs: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.ObjectMeta.Name = scenarioProps.expectedNewPvName - pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName - setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) - pv.Spec.ClaimRef.Name = scenarioProps.expectedNewPvcName - pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - changedNewRadixVolumeName: "volume101", - changedNewRadixVolumeStorageName: "storage101", - expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - expectedNewSecretName: "some-component-volume101-csiazurecreds", - expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - expectedNewPvName: "pv-radixvolumemount-some-uuid", - }), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) - pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) - pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) - matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - return deploymentVolumesTestScenario{ - name: "Garbage collect orphaned PVCs and PersistentVolume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcs: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - expectedPvcs: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - pvcForAnotherNamespace, - pvcForAnotherComponent, - }, - existingPvs: []corev1.PersistentVolume{ - createRandomPv(props, props.namespace, props.componentName), - pvForAnotherNamespace, - pvForAnotherComponent, - }, - expectedPvs: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - pvForAnotherNamespace, - pvForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Set readonly volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcs: []corev1.PersistentVolumeClaim{ - createRandomPvc(props, props.namespace, props.componentName), - }, - expectedPvcs: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - }), - }, - existingPvs: []corev1.PersistentVolume{ - createRandomPv(props, props.namespace, props.componentName), - }, - expectedPvs: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadWriteOnce volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcs: []corev1.PersistentVolumeClaim{ - existingPvc, - }, - expectedPvcs: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - }), - }, - existingPvs: []corev1.PersistentVolume{ - existingPv, - }, - expectedPvs: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadWriteMany volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcs: []corev1.PersistentVolumeClaim{ - existingPvc, - }, - expectedPvcs: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - }), - }, - existingPvs: []corev1.PersistentVolume{ - existingPv, - }, - expectedPvs: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - existingPvc := createExpectedPvc(props, nil) - existingPv := createExpectedPv(props, nil) - matchPvAndPvc(&existingPv, &existingPvc) - return deploymentVolumesTestScenario{ - name: "Set ReadOnlyMany volume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcs: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - }), - }, - expectedPvcs: []corev1.PersistentVolumeClaim{ - modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - }), - }, - existingPvs: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - pv.Spec.MountOptions = getMountOptions(props, false) - }), - }, - expectedPvs: []corev1.PersistentVolume{ - modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - pv.Spec.MountOptions = getMountOptions(props, true) - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcs: []corev1.PersistentVolumeClaim{}, - expectedPvcs: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPvs: []corev1.PersistentVolume{}, - expectedPvs: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcs: []corev1.PersistentVolumeClaim{}, - expectedPvcs: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPvs: []corev1.PersistentVolume{}, - expectedPvs: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, - "--streaming=true", - "--stream-cache-mb=101", - "--block-size-mb=102", - "--buffer-size-mb=103", - "--max-buffers=104", - "--max-blocks-per-file=105", - "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) - - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - return deploymentVolumesTestScenario{ - name: "Create new BlobFuse2 volume has disabled streaming", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - Enabled: pointers.Ptr(false), - StreamCache: pointers.Ptr(uint64(101)), - BlockSize: pointers.Ptr(uint64(102)), - BufferSize: pointers.Ptr(uint64(103)), - MaxBuffers: pointers.Ptr(uint64(104)), - MaxBlocksPerFile: pointers.Ptr(uint64(105)), - } - }), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcs: []corev1.PersistentVolumeClaim{}, - expectedPvcs: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - }, - existingPvs: []corev1.PersistentVolume{}, - expectedPvs: []corev1.PersistentVolume{ - createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, - "--use-adls=false") - }), - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - } - }()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{}, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPvs: []corev1.PersistentVolume{}, + // expectedPvs: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // type scenarioProperties struct { + // changedNewRadixVolumeName string + // changedNewRadixVolumeStorageName string + // expectedVolumeName string + // expectedNewSecretName string + // expectedNewPvcName string + // expectedNewPvName string + // } + // getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Update storage in existing volume name and storage", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.Name = scenarioProps.changedNewRadixVolumeName + // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props, func(v *corev1.Volume) { + // v.Name = scenarioProps.expectedVolumeName + // }), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + // }), + // }, + // existingPvs: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // }, + // expectedPvs: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + // pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName + // setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + // pv.Spec.ClaimRef.Name = scenarioProps.expectedNewPvcName + // pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + // changedNewRadixVolumeName: "volume101", + // changedNewRadixVolumeStorageName: "storage101", + // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + // expectedNewSecretName: "some-component-volume101-csiazurecreds", + // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + // expectedNewPvName: "pv-radixvolumemount-some-uuid", + // }), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) + // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + // pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) + // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + // matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // return deploymentVolumesTestScenario{ + // name: "Garbage collect orphaned PVCs and PersistentVolume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, + // }, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherNamespace, + // pvcForAnotherComponent, + // }, + // existingPvs: []corev1.PersistentVolume{ + // createRandomPv(props, props.namespace, props.componentName), + // pvForAnotherNamespace, + // pvForAnotherComponent, + // }, + // expectedPvs: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // pvForAnotherNamespace, + // pvForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Set readonly volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{ + // createRandomPvc(props, props.namespace, props.componentName), + // }, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // }), + // }, + // existingPvs: []corev1.PersistentVolume{ + // createRandomPv(props, props.namespace, props.componentName), + // }, + // expectedPvs: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteOnce volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{ + // existingPvc, + // }, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // }), + // }, + // existingPvs: []corev1.PersistentVolume{ + // existingPv, + // }, + // expectedPvs: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadWriteMany volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{ + // existingPvc, + // }, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // }), + // }, + // existingPvs: []corev1.PersistentVolume{ + // existingPv, + // }, + // expectedPvs: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createExpectedPvc(props, nil) + // existingPv := createExpectedPv(props, nil) + // matchPvAndPvc(&existingPv, &existingPvc) + // return deploymentVolumesTestScenario{ + // name: "Set ReadOnlyMany volume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // }), + // }, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // }), + // }, + // existingPvs: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }), + // }, + // expectedPvs: []corev1.PersistentVolume{ + // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // pv.Spec.MountOptions = getMountOptions(props, true) + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{}, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPvs: []corev1.PersistentVolume{}, + // expectedPvs: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + // StreamCache: pointers.Ptr(uint64(101)), + // BlockSize: pointers.Ptr(uint64(102)), + // BufferSize: pointers.Ptr(uint64(103)), + // MaxBuffers: pointers.Ptr(uint64(104)), + // MaxBlocksPerFile: pointers.Ptr(uint64(105)), + // } + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{}, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPvs: []corev1.PersistentVolume{}, + // expectedPvs: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, + // "--streaming=true", + // "--stream-cache-mb=101", + // "--block-size-mb=102", + // "--buffer-size-mb=103", + // "--max-buffers=104", + // "--max-blocks-per-file=105", + // "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) + // + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // return deploymentVolumesTestScenario{ + // name: "Create new BlobFuse2 volume has disabled streaming", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + // Enabled: pointers.Ptr(false), + // StreamCache: pointers.Ptr(uint64(101)), + // BlockSize: pointers.Ptr(uint64(102)), + // BufferSize: pointers.Ptr(uint64(103)), + // MaxBuffers: pointers.Ptr(uint64(104)), + // MaxBlocksPerFile: pointers.Ptr(uint64(105)), + // } + // }), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{}, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // }, + // existingPvs: []corev1.PersistentVolume{}, + // expectedPvs: []corev1.PersistentVolume{ + // createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // pv.Spec.MountOptions = getMountOptions(props, true, + // "--use-adls=false") + // }), + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + // } + //}()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) - pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) + pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) + pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) {}) + anotherComponentVolume := createRandomVolume(func(volume *corev1.Volume) { volume.PersistentVolumeClaim.ClaimName = pvcForAnotherComponent.Name }) + volume := createTestVolume(props) + existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) { pv.Spec.ClaimRef.Name = volume.PersistentVolumeClaim.ClaimName }) + expectedPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) + expectedPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + matchPvAndPvc(&expectedPv, &expectedPvc) return deploymentVolumesTestScenario{ name: "Do not change existing PersistentVolume with class name, when creating new PVC", props: props, radixVolumeMounts: []v1.RadixVolumeMount{ + createRandomVolumeMount(func(vm *v1.RadixVolumeMount) { vm.Name = anotherVolumeMountName }), createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), }, volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcs: []corev1.PersistentVolumeClaim{ - pvcForAnotherComponent, - }, - expectedPvcs: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - pvcForAnotherComponent, - }, - existingPvs: []corev1.PersistentVolume{ - existingPv, - pvForAnotherComponent, - }, - expectedPvs: []corev1.PersistentVolume{ - existingPv, - pvForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) - return deploymentVolumesTestScenario{ - name: "Do not change existing PersistentVolume without class name, when creating new PVC", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), + anotherComponentVolume, + volume, }, existingPvcs: []corev1.PersistentVolumeClaim{ pvcForAnotherComponent, }, expectedPvcs: []corev1.PersistentVolumeClaim{ - createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + expectedPvc, pvcForAnotherComponent, }, existingPvs: []corev1.PersistentVolume{ @@ -947,47 +919,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { pvForAnotherComponent, }, expectedPvs: []corev1.PersistentVolume{ - existingPv, - pvForAnotherComponent, - }, - } - } - return []deploymentVolumesTestScenario{ - getScenario(getPropsCsiBlobVolume1Storage1(nil)), - } - }()...) - scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName) - pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName) - matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) - expectedPvc := createRandomPvc(props, props.namespace, componentName) - expectedPv := createRandomPv(props, props.namespace, componentName) - matchPvAndPvc(&expectedPv, &expectedPvc) - return deploymentVolumesTestScenario{ - name: "Do not change existing PVC with class name, when creating new PersistentVolume", - props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - }, - volumes: []corev1.Volume{ - createTestVolume(props, func(v *corev1.Volume) {}), - }, - existingPvcs: []corev1.PersistentVolumeClaim{ - pvcForAnotherComponent, - existingPvc, - }, - expectedPvcs: []corev1.PersistentVolumeClaim{ - pvcForAnotherComponent, - expectedPvc, - }, - existingPvs: []corev1.PersistentVolume{ - pvForAnotherComponent, - }, - expectedPvs: []corev1.PersistentVolume{ - pvForAnotherComponent, expectedPv, + pvForAnotherComponent, }, } } @@ -995,28 +928,121 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { getScenario(getPropsCsiBlobVolume1Storage1(nil)), } }()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + // return deploymentVolumesTestScenario{ + // name: "Do not change existing PersistentVolume without class name, when creating new PVC", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{ + // pvcForAnotherComponent, + // }, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // pvcForAnotherComponent, + // }, + // existingPvs: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // expectedPvs: []corev1.PersistentVolume{ + // existingPv, + // pvForAnotherComponent, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) + // pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) + // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + // existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) + // expectedPvc := createRandomPvc(props, props.namespace, componentName) + // expectedPv := createRandomPv(props, props.namespace, componentName) + // matchPvAndPvc(&expectedPv, &expectedPvc) + // return deploymentVolumesTestScenario{ + // name: "Do not change existing PVC with class name, when creating new PersistentVolume", + // props: props, + // radixVolumeMounts: []v1.RadixVolumeMount{ + // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + // }, + // volumes: []corev1.Volume{ + // createTestVolume(props), + // }, + // existingPvcs: []corev1.PersistentVolumeClaim{ + // pvcForAnotherComponent, + // existingPvc, + // }, + // expectedPvcs: []corev1.PersistentVolumeClaim{ + // pvcForAnotherComponent, + // expectedPvc, + // }, + // existingPvs: []corev1.PersistentVolume{ + // pvForAnotherComponent, + // }, + // expectedPvs: []corev1.PersistentVolume{ + // pvForAnotherComponent, + // expectedPv, + // }, + // } + // } + // return []deploymentVolumesTestScenario{ + // getScenario(getPropsCsiBlobVolume1Storage1(nil)), + // } + //}()...) + + //suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { + //TODO uncomment: for _, factory := range suite.radixCommonDeployComponentFactories[] { + t := suite.T() + factory := suite.radixCommonDeployComponentFactories[0] + scenario := scenarios[0] + + //for _, factory := range suite.radixCommonDeployComponentFactories[:1] { + // for _, scenario := range scenarios { + t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) + testEnv := getTestEnv() + radixDeployment := buildRd(appName, environment, componentName, scenario.radixVolumeMounts) + putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeUtil.KubeClient(), &scenario) + desiredVolumes := getDesiredDeployment(componentName, scenario.volumes).Spec.Template.Spec.Volumes + + deployComponent := radixDeployment.Spec.Components[0] + actualVolumes, err := CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, utils.GetEnvironmentNamespace(appName, environment), &deployComponent, desiredVolumes) + require.NoError(t, err) + assert.Equal(t, len(scenario.volumes), len(actualVolumes), "Number of volumes is not equal") + + existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) + require.NoError(t, err) + assert.True(t, equalPersistentVolumeClaims(&scenario.expectedPvcs, &existingPvcs), "PVC-s are not equal") + assert.True(t, equalPersistentVolumes(&scenario.expectedPvs, &existingPvs), "PV-s are not equal") + // } + //} + //}) +} - suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { - for _, factory := range suite.radixCommonDeployComponentFactories[:1] { - for _, scenario := range scenarios { - t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) - testEnv := getTestEnv() - radixDeployment := buildRd(appName, environment, componentName, scenario.radixVolumeMounts) - putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeUtil.KubeClient(), &scenario) - desiredVolumes := getDesiredDeployment(componentName, scenario.volumes).Spec.Template.Spec.Volumes - - deployComponent := radixDeployment.Spec.Components[0] - actualVolumes, err := CreateOrUpdateCsiAzureVolumeResources(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, utils.GetEnvironmentNamespace(appName, environment), &deployComponent, desiredVolumes) - require.NoError(t, err) - assert.Equal(t, len(scenario.volumes), len(actualVolumes), "Number of volumes is not equal") - - existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) - require.NoError(t, err) - assert.True(t, equalPersistentVolumeClaims(&scenario.expectedPvcs, &existingPvcs), "PVC-s are not equal") - assert.True(t, equalPersistentVolumes(&scenario.expectedPvs, &existingPvs), "PV-s are not equal") - } - } - }) +func createRandomVolumeMount(modify func(mount *v1.RadixVolumeMount)) v1.RadixVolumeMount { + vm := v1.RadixVolumeMount{ + Name: strings.ToLower(utils.RandString(10)), + Path: "/tmp/" + strings.ToLower(utils.RandString(10)), + BlobFuse2: &v1.RadixBlobFuse2VolumeMount{ + Protocol: v1.BlobFuse2ProtocolFuse2, + Container: strings.ToLower(utils.RandString(10)), + }, + } + modify(&vm) + return vm } func matchPvAndPvc(pv *corev1.PersistentVolume, pvc *corev1.PersistentVolumeClaim) { @@ -1033,7 +1059,7 @@ func matchPvAndPvc(pv *corev1.PersistentVolume, pvc *corev1.PersistentVolumeClai func createRandomPv(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolume { return createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pvName := getCsiAzurePersistentVolumeName() + pvName := getCsiAzurePvName() pv.ObjectMeta.Name = pvName pv.ObjectMeta.Labels[kube.RadixNamespace] = namespace pv.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName @@ -1043,11 +1069,11 @@ func createRandomPv(props expectedPvcPvProperties, namespace, componentName stri func createRandomPvc(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolumeClaim { return createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvcName, err := getCsiAzurePersistentVolumeClaimName(componentName, &v1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: v1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.radixStorageName, Path: "/tmp"}) + pvcName, err := getCsiAzurePvcName(componentName, &v1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: v1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.radixStorageName, Path: "/tmp"}) if err != nil { panic(err) } - pvName := getCsiAzurePersistentVolumeName() + pvName := getCsiAzurePvName() pvc.ObjectMeta.Name = pvcName pvc.ObjectMeta.Namespace = namespace pvc.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName @@ -1055,7 +1081,7 @@ func createRandomPvc(props expectedPvcPvProperties, namespace, componentName str }) } -func createRandomAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolume { +func createRandomAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties, namespace, componentName, anotherVolumeMountName string) corev1.PersistentVolume { return createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) { pvName := "pvc-" + uuid.NewString() pv.ObjectMeta.Name = pvName @@ -1064,20 +1090,22 @@ func createRandomAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties } pv.ObjectMeta.Labels[kube.RadixNamespace] = namespace pv.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName + pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = anotherVolumeMountName pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvName] = pvName }) } -func createRandomAutoProvisionedPvcWithStorageClass(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolumeClaim { +func createRandomAutoProvisionedPvcWithStorageClass(props expectedPvcPvProperties, namespace, componentName, anotherVolumeMountName string) corev1.PersistentVolumeClaim { return createExpectedAutoProvisionedPvcWithStorageClass(props, func(pvc *corev1.PersistentVolumeClaim) { - pvcName, err := getCsiAzurePersistentVolumeClaimName(componentName, &v1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: v1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.radixStorageName, Path: "/tmp"}) + pvcName, err := getCsiAzurePvcName(componentName, &v1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: v1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.radixStorageName, Path: "/tmp"}) if err != nil { panic(err) } - pvName := getCsiAzurePersistentVolumeName() + pvName := getCsiAzurePvName() pvc.ObjectMeta.Name = pvcName pvc.ObjectMeta.Namespace = namespace pvc.ObjectMeta.Labels[kube.RadixComponentLabel] = componentName + pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = anotherVolumeMountName pvc.Spec.VolumeName = pvName }) } @@ -1465,13 +1493,22 @@ func createExpectedAutoProvisionedPvcWithStorageClass(props expectedPvcPvPropert return pvc } -func createTestVolume(pvcProps expectedPvcPvProperties, modify func(*corev1.Volume)) corev1.Volume { - volume := corev1.Volume{ +func createTestVolume(pvcProps expectedPvcPvProperties) corev1.Volume { + return corev1.Volume{ Name: pvcProps.volumeName, VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: pvcProps.pvcName, }}, } +} + +func createRandomVolume(modify func(*corev1.Volume)) corev1.Volume { + volume := corev1.Volume{ + Name: strings.ToLower(utils.RandString(10)), + VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: strings.ToLower(utils.RandString(10)), + }}, + } if modify != nil { modify(&volume) } From b20aa7c615581ad96ecc2d0645f8b845227d98fe Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 14 Jan 2025 10:06:46 +0100 Subject: [PATCH 61/68] Correcting unit-tests --- pkg/apis/volumemount/volumemount.go | 7 +- pkg/apis/volumemount/volumemount_test.go | 720 +++++++++++------------ 2 files changed, 354 insertions(+), 373 deletions(-) diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index 20ba9a3da..a8aa17cb2 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -25,12 +25,13 @@ import ( ) const ( + // ReadOnlyMountOption The readonly volume mount option for CSI fuse driver + ReadOnlyMountOption = "-o ro" csiVolumeTypeBlobFuse2ProtocolFuse = "csi-az-blob" csiVolumeTypeBlobFuse2ProtocolFuse2 = "csi-blobfuse2-fuse2" csiVolumeNameTemplate = "%s-%s-%s-%s" // --- csiAzureKeyVaultSecretMountPathTemplate = "/mnt/azure-key-vault/%s" - - volumeNameMaxLength = 63 + volumeNameMaxLength = 63 ) // These are valid volume mount provisioners @@ -627,7 +628,7 @@ func getCsiAzurePvMountOptionsForAzureBlob(radixVolumeMount *radixv1.RadixVolume } } if getVolumeMountAccessMode(radixVolumeMount) == corev1.ReadOnlyMany { - mountOptions = append(mountOptions, "-o ro") + mountOptions = append(mountOptions, ReadOnlyMountOption) } if radixVolumeMount.BlobFuse2 != nil { mountOptions = append(mountOptions, getStreamingMountOptions(radixVolumeMount.BlobFuse2.Streaming)...) diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index 042fca4f6..d56f929c7 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -3,23 +3,22 @@ package volumemount import ( "context" "fmt" - "github.com/equinor/radix-operator/pkg/apis/defaults" - "github.com/equinor/radix-operator/pkg/apis/defaults/k8s" - "github.com/equinor/radix-operator/pkg/apis/internal" - "github.com/google/uuid" - "k8s.io/apimachinery/pkg/types" "strings" "testing" "time" commonUtils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/pointers" + "github.com/equinor/radix-operator/pkg/apis/defaults" + "github.com/equinor/radix-operator/pkg/apis/defaults/k8s" + "github.com/equinor/radix-operator/pkg/apis/internal" "github.com/equinor/radix-operator/pkg/apis/internal/persistentvolume" "github.com/equinor/radix-operator/pkg/apis/kube" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" radix "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" + "github.com/google/uuid" kedav2 "github.com/kedacore/keda/v2/pkg/generated/clientset/versioned" kedafake "github.com/kedacore/keda/v2/pkg/generated/clientset/versioned/fake" prometheusclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" @@ -31,6 +30,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" kubefake "k8s.io/client-go/kubernetes/fake" secretProviderClient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" @@ -77,6 +77,18 @@ type pvcTestScenario struct { pvc corev1.PersistentVolumeClaim } +const ( + appName = "any-app" + environment = "some-env" + componentName = "some-component" +) + +var ( + anotherNamespace = strings.ToLower(commonUtils.RandString(10)) + anotherComponentName = strings.ToLower(commonUtils.RandString(10)) + anotherVolumeMountName = strings.ToLower(commonUtils.RandString(10)) +) + func TestVolumeMountTestSuite(t *testing.T) { suite.Run(t, new(VolumeMountTestSuite)) } @@ -490,15 +502,6 @@ func modifyPv(pv corev1.PersistentVolume, modify func(pv *corev1.PersistentVolum } func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { - const ( - appName = "any-app" - environment = "some-env" - componentName = "some-component" - ) - //anotherNamespace := strings.ToLower(commonUtils.RandString(10)) - anotherComponentName := strings.ToLower(commonUtils.RandString(10)) - anotherVolumeMountName := strings.ToLower(commonUtils.RandString(10)) - var scenarios []deploymentVolumesTestScenario //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { @@ -509,7 +512,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), // }, // volumes: []corev1.Volume{ - // createTestVolume(props), + // createTestVolume(props, nil), // }, // existingPvcs: []corev1.PersistentVolumeClaim{}, // expectedPvcs: []corev1.PersistentVolumeClaim{ @@ -535,6 +538,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // expectedNewPvName string // } // getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + // existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + // existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) // return deploymentVolumesTestScenario{ // name: "Update storage in existing volume name and storage", // props: props, @@ -550,9 +555,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }), // }, // existingPvcs: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + // existingPvc, // }, // expectedPvcs: []corev1.PersistentVolumeClaim{ + // existingPvc, // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName @@ -560,9 +566,10 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // }), // }, // existingPvs: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + // existingPv, // }, // expectedPvs: []corev1.PersistentVolume{ + // existingPv, // createExpectedPv(props, func(pv *corev1.PersistentVolume) { // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName @@ -587,49 +594,14 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { //}()...) //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherNamespace := createRandomPv(props, anotherNamespace, anotherComponentName) - // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - // pvcForAnotherNamespace := createRandomPvc(props, anotherNamespace, anotherComponentName) - // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - // matchPvAndPvc(&pvForAnotherNamespace, &pvcForAnotherNamespace) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // return deploymentVolumesTestScenario{ - // name: "Garbage collect orphaned PVCs and PersistentVolume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, - // }, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherNamespace, - // pvcForAnotherComponent, - // }, - // existingPvs: []corev1.PersistentVolume{ - // createRandomPv(props, props.namespace, props.componentName), - // pvForAnotherNamespace, - // pvForAnotherComponent, - // }, - // expectedPvs: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // pvForAnotherNamespace, - // pvForAnotherComponent, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + // existingPvc := createRandomPvc(props, props.namespace, props.componentName) + // expectedPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + // }) + // existingPv := createRandomPv(props, props.namespace, props.componentName) + // expectedPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) { + // //pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") + // }) // return deploymentVolumesTestScenario{ // name: "Set readonly volume", // props: props, @@ -637,23 +609,21 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), // }, // volumes: []corev1.Volume{ - // createTestVolume(props), + // createTestVolume(props, func(v *corev1.Volume) {}), // }, // existingPvcs: []corev1.PersistentVolumeClaim{ - // createRandomPvc(props, props.namespace, props.componentName), + // existingPvc, // }, // expectedPvcs: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // }), + // existingPvc, + // expectedPvc, // }, // existingPvs: []corev1.PersistentVolume{ - // createRandomPv(props, props.namespace, props.componentName), + // existingPv, // }, // expectedPvs: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - // }), + // existingPv, + // expectedPv, // }, // } // } @@ -666,6 +636,13 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // existingPvc := createExpectedPvc(props, nil) // existingPv := createExpectedPv(props, nil) // matchPvAndPvc(&existingPv, &existingPvc) + // expectedPv := modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // pv.Spec.MountOptions = getMountOptions(props, false) + // }) + // expectedPvc := modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + // }) // return deploymentVolumesTestScenario{ // name: "Set ReadWriteOnce volume", // props: props, @@ -673,103 +650,21 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), // }, // volumes: []corev1.Volume{ - // createTestVolume(props), + // createTestVolume(props, func(v *corev1.Volume) {}), // }, // existingPvcs: []corev1.PersistentVolumeClaim{ // existingPvc, // }, // expectedPvcs: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // }), - // }, - // existingPvs: []corev1.PersistentVolume{ - // existingPv, - // }, - // expectedPvs: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteMany volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{ // existingPvc, - // }, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // }), + // expectedPvc, // }, // existingPvs: []corev1.PersistentVolume{ // existingPv, // }, // expectedPvs: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // return deploymentVolumesTestScenario{ - // name: "Set ReadOnlyMany volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // }), - // }, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // }), - // }, - // existingPvs: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }), - // }, - // expectedPvs: []corev1.PersistentVolume{ - // modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // pv.Spec.MountOptions = getMountOptions(props, true) - // }), + // existingPv, + // expectedPv, // }, // } // } @@ -777,121 +672,205 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { // getScenario(getPropsCsiBlobVolume1Storage1(nil)), // } //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{}, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPvs: []corev1.PersistentVolume{}, - // expectedPvs: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - // StreamCache: pointers.Ptr(uint64(101)), - // BlockSize: pointers.Ptr(uint64(102)), - // BufferSize: pointers.Ptr(uint64(103)), - // MaxBuffers: pointers.Ptr(uint64(104)), - // MaxBlocksPerFile: pointers.Ptr(uint64(105)), - // } - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{}, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPvs: []corev1.PersistentVolume{}, - // expectedPvs: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, - // "--streaming=true", - // "--stream-cache-mb=101", - // "--block-size-mb=102", - // "--buffer-size-mb=103", - // "--max-buffers=104", - // "--max-blocks-per-file=105", - // "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) - // - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new BlobFuse2 volume has disabled streaming", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ - // Enabled: pointers.Ptr(false), - // StreamCache: pointers.Ptr(uint64(101)), - // BlockSize: pointers.Ptr(uint64(102)), - // BufferSize: pointers.Ptr(uint64(103)), - // MaxBuffers: pointers.Ptr(uint64(104)), - // MaxBlocksPerFile: pointers.Ptr(uint64(105)), - // } - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{}, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPvs: []corev1.PersistentVolume{}, - // expectedPvs: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.Spec.MountOptions = getMountOptions(props, true, - // "--use-adls=false") - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), - // } - //}()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteMany volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }), + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pv.Spec.MountOptions = getMountOptions(props, false) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + existingPv = modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + pv.Spec.MountOptions = getMountOptions(props, false) + }) + existingPvc = modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }) + return deploymentVolumesTestScenario{ + name: "Set ReadOnlyMany volume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }), + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + pv.Spec.MountOptions = getMountOptions(props, true) + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, + "--streaming=true", + "--stream-cache-mb=101", + "--block-size-mb=102", + "--buffer-size-mb=103", + "--max-buffers=104", + "--max-blocks-per-file=105", + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) + + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new BlobFuse2 volume has disabled streaming", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + Enabled: pointers.Ptr(false), + StreamCache: pointers.Ptr(uint64(101)), + BlockSize: pointers.Ptr(uint64(102)), + BufferSize: pointers.Ptr(uint64(103)), + MaxBuffers: pointers.Ptr(uint64(104)), + MaxBlocksPerFile: pointers.Ptr(uint64(105)), + } + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = getMountOptions(props, true, + "--use-adls=false") + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobFuse2Volume1Storage1(nil)), + } + }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - anotherComponentVolume := createRandomVolume(func(volume *corev1.Volume) { volume.PersistentVolumeClaim.ClaimName = pvcForAnotherComponent.Name }) - volume := createTestVolume(props) + volume := createTestVolume(props, func(v *corev1.Volume) {}) existingPv := createAutoProvisionedPvWithStorageClass(props, func(pv *corev1.PersistentVolume) { pv.Spec.ClaimRef.Name = volume.PersistentVolumeClaim.ClaimName }) expectedPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) expectedPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) @@ -904,7 +883,6 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), }, volumes: []corev1.Volume{ - anotherComponentVolume, volume, }, existingPvcs: []corev1.PersistentVolumeClaim{ @@ -920,6 +898,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }, expectedPvs: []corev1.PersistentVolume{ expectedPv, + existingPv, pvForAnotherComponent, }, } @@ -928,108 +907,105 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { getScenario(getPropsCsiBlobVolume1Storage1(nil)), } }()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) - // pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) - // return deploymentVolumesTestScenario{ - // name: "Do not change existing PersistentVolume without class name, when creating new PVC", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{ - // pvcForAnotherComponent, - // }, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // pvcForAnotherComponent, - // }, - // existingPvs: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // expectedPvs: []corev1.PersistentVolume{ - // existingPv, - // pvForAnotherComponent, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) - // pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) - // matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) - // existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) - // expectedPvc := createRandomPvc(props, props.namespace, componentName) - // expectedPv := createRandomPv(props, props.namespace, componentName) - // matchPvAndPvc(&expectedPv, &expectedPvc) - // return deploymentVolumesTestScenario{ - // name: "Do not change existing PVC with class name, when creating new PersistentVolume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{ - // pvcForAnotherComponent, - // existingPvc, - // }, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // pvcForAnotherComponent, - // expectedPvc, - // }, - // existingPvs: []corev1.PersistentVolume{ - // pvForAnotherComponent, - // }, - // expectedPvs: []corev1.PersistentVolume{ - // pvForAnotherComponent, - // expectedPv, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomPv(props, props.namespace, anotherComponentName) + pvcForAnotherComponent := createRandomPvc(props, props.namespace, anotherComponentName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + return deploymentVolumesTestScenario{ + name: "Do not change existing PersistentVolume without class name, when creating new PVC", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + pvcForAnotherComponent, + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + pvForAnotherComponent, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + pvForAnotherComponent := createRandomAutoProvisionedPvWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) + pvcForAnotherComponent := createRandomAutoProvisionedPvcWithStorageClass(props, props.namespace, anotherComponentName, anotherVolumeMountName) + matchPvAndPvc(&pvForAnotherComponent, &pvcForAnotherComponent) + existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) + expectedPvc := createRandomPvc(props, props.namespace, componentName) + expectedPv := createRandomPv(props, props.namespace, componentName) + matchPvAndPvc(&expectedPv, &expectedPvc) + return deploymentVolumesTestScenario{ + name: "Do not change existing PVC with class name, when creating new PersistentVolume", + props: props, + radixVolumeMounts: []v1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + pvcForAnotherComponent, + expectedPvc, + }, + existingPvs: []corev1.PersistentVolume{ + pvForAnotherComponent, + }, + expectedPvs: []corev1.PersistentVolume{ + pvForAnotherComponent, + expectedPv, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) - //suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { - //TODO uncomment: for _, factory := range suite.radixCommonDeployComponentFactories[] { - t := suite.T() - factory := suite.radixCommonDeployComponentFactories[0] - scenario := scenarios[0] - - //for _, factory := range suite.radixCommonDeployComponentFactories[:1] { - // for _, scenario := range scenarios { - t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) - testEnv := getTestEnv() - radixDeployment := buildRd(appName, environment, componentName, scenario.radixVolumeMounts) - putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeUtil.KubeClient(), &scenario) - desiredVolumes := getDesiredDeployment(componentName, scenario.volumes).Spec.Template.Spec.Volumes - - deployComponent := radixDeployment.Spec.Components[0] - actualVolumes, err := CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, utils.GetEnvironmentNamespace(appName, environment), &deployComponent, desiredVolumes) - require.NoError(t, err) - assert.Equal(t, len(scenario.volumes), len(actualVolumes), "Number of volumes is not equal") - - existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) - require.NoError(t, err) - assert.True(t, equalPersistentVolumeClaims(&scenario.expectedPvcs, &existingPvcs), "PVC-s are not equal") - assert.True(t, equalPersistentVolumes(&scenario.expectedPvs, &existingPvs), "PV-s are not equal") - // } - //} - //}) + suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { + for _, factory := range suite.radixCommonDeployComponentFactories { + for _, scenario := range scenarios { + t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) + testEnv := getTestEnv() + radixDeployment := buildRd(appName, environment, componentName, scenario.radixVolumeMounts) + putExistingDeploymentVolumesScenarioDataToFakeCluster(testEnv.kubeUtil.KubeClient(), &scenario) + desiredVolumes := getDesiredDeployment(componentName, scenario.volumes).Spec.Template.Spec.Volumes + + deployComponent := radixDeployment.Spec.Components[0] + actualVolumes, err := CreateOrUpdateCsiAzureVolumeResourcesForDeployComponent(context.Background(), testEnv.kubeUtil.KubeClient(), radixDeployment, utils.GetEnvironmentNamespace(appName, environment), &deployComponent, desiredVolumes) + require.NoError(t, err) + assert.Equal(t, len(scenario.volumes), len(actualVolumes), "Number of volumes is not equal") + + existingPvcs, existingPvs, err := getExistingPvcsAndPersistentVolumeFromFakeCluster(testEnv.kubeUtil.KubeClient()) + require.NoError(t, err) + assert.Len(t, existingPvcs, len(scenario.expectedPvcs), "PVC-s count is not equal") + assert.True(t, equalPersistentVolumeClaims(&scenario.expectedPvcs, &existingPvcs), "PVC-s are not equal") + assert.Len(t, existingPvs, len(scenario.expectedPvs), "PV-s count is not equal") + assert.True(t, equalPersistentVolumes(&scenario.expectedPvs, &existingPvs), "PV-s are not equal") + } + } + }) } func createRandomVolumeMount(modify func(mount *v1.RadixVolumeMount)) v1.RadixVolumeMount { @@ -1371,7 +1347,7 @@ func getMountOptions(props expectedPvcPvProperties, readOnly bool, extraOptions "-o negative_timeout=120", } if readOnly { - options = append(options, "-o ro") + options = append(options, ReadOnlyMountOption) } idOption := getPersistentVolumeIdMountOption(props) if len(idOption) > 0 { @@ -1395,7 +1371,7 @@ func getMountOptionsInRandomOrder(props expectedPvcPvProperties, readOnly bool, options = append(options, idOption) } if readOnly { - options = append(options, "-o ro") + options = append(options, ReadOnlyMountOption) } return append(options, extraOptions...) } @@ -1493,13 +1469,17 @@ func createExpectedAutoProvisionedPvcWithStorageClass(props expectedPvcPvPropert return pvc } -func createTestVolume(pvcProps expectedPvcPvProperties) corev1.Volume { - return corev1.Volume{ +func createTestVolume(pvcProps expectedPvcPvProperties, modify func(v *corev1.Volume)) corev1.Volume { + volume := corev1.Volume{ Name: pvcProps.volumeName, VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: pvcProps.pvcName, }}, } + if modify != nil { + modify(&volume) + } + return volume } func createRandomVolume(modify func(*corev1.Volume)) corev1.Volume { From c05a0b82f789b2bdb473116611605bedd5e3ab54 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 14 Jan 2025 10:39:12 +0100 Subject: [PATCH 62/68] Correcting unit-tests --- pkg/apis/volumemount/volumemount_test.go | 519 ++++++++++++----------- 1 file changed, 260 insertions(+), 259 deletions(-) diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index d56f929c7..a923f5812 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -14,10 +14,10 @@ import ( "github.com/equinor/radix-operator/pkg/apis/internal" "github.com/equinor/radix-operator/pkg/apis/internal/persistentvolume" "github.com/equinor/radix-operator/pkg/apis/kube" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" - radix "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" + radixfake "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" "github.com/google/uuid" kedav2 "github.com/kedacore/keda/v2/pkg/generated/clientset/versioned" kedafake "github.com/kedacore/keda/v2/pkg/generated/clientset/versioned/fake" @@ -39,7 +39,7 @@ import ( type VolumeMountTestSuite struct { suite.Suite - radixCommonDeployComponentFactories []v1.RadixCommonDeployComponentFactory + radixCommonDeployComponentFactories []radixv1.RadixCommonDeployComponentFactory } type TestEnv struct { @@ -53,7 +53,7 @@ type TestEnv struct { type volumeMountTestScenario struct { name string - radixVolumeMount v1.RadixVolumeMount + radixVolumeMount radixv1.RadixVolumeMount expectedVolumeName string expectedVolumeNameIsPrefix bool expectedError string @@ -63,7 +63,7 @@ type volumeMountTestScenario struct { type deploymentVolumesTestScenario struct { name string props expectedPvcPvProperties - radixVolumeMounts []v1.RadixVolumeMount + radixVolumeMounts []radixv1.RadixVolumeMount volumes []corev1.Volume existingPvs []corev1.PersistentVolume existingPvcs []corev1.PersistentVolumeClaim @@ -94,16 +94,16 @@ func TestVolumeMountTestSuite(t *testing.T) { } func (suite *VolumeMountTestSuite) SetupSuite() { - suite.radixCommonDeployComponentFactories = []v1.RadixCommonDeployComponentFactory{ - v1.RadixDeployComponentFactory{}, - v1.RadixDeployJobComponentFactory{}, + suite.radixCommonDeployComponentFactories = []radixv1.RadixCommonDeployComponentFactory{ + radixv1.RadixDeployComponentFactory{}, + radixv1.RadixDeployJobComponentFactory{}, } } func getTestEnv() TestEnv { testEnv := TestEnv{ kubeclient: kubefake.NewSimpleClientset(), - radixclient: radix.NewSimpleClientset(), + radixclient: radixfake.NewSimpleClientset(), kedaClient: kedafake.NewSimpleClientset(), secretproviderclient: secretproviderfake.NewSimpleClientset(), prometheusclient: prometheusfake.NewSimpleClientset(), @@ -131,15 +131,15 @@ func (suite *VolumeMountTestSuite) Test_NoVolumeMounts() { func (suite *VolumeMountTestSuite) Test_ValidBlobCsiAzureVolumeMounts() { scenarios := []volumeMountTestScenario{ { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storageName1", Path: "TestPath1"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storageName1", Path: "TestPath1"}, expectedVolumeName: "csi-az-blob-app-volume1-storageName1", }, { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume2", Storage: "storageName2", Path: "TestPath2"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume2", Storage: "storageName2", Path: "TestPath2"}, expectedVolumeName: "csi-az-blob-app-volume2-storageName2", }, { - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "TestPath2"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "TestPath2"}, expectedVolumeName: "csi-az-blob-app-volume-with-long-name-storageName-with-lo-", expectedVolumeNameIsPrefix: true, }, @@ -194,17 +194,17 @@ func (suite *VolumeMountTestSuite) Test_FailBlobCsiAzureVolumeMounts() { scenarios := []volumeMountTestScenario{ { name: "Missed volume mount name", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Storage: "storageName1", Path: "TestPath1"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Storage: "storageName1", Path: "TestPath1"}, expectedError: "name is empty for volume mount in the component app", }, { name: "Missed volume mount storage", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Path: "TestPath1"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Path: "TestPath1"}, expectedError: "storage is empty for volume mount volume1 in the component app", }, { name: "Missed volume mount path", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storageName1"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storageName1"}, expectedError: "path is empty for volume mount volume1 in the component app", }, } @@ -240,13 +240,13 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { scenarios := []volumeMountTestScenario{ { name: "Blob CSI Azure volume", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storage1", Path: "path1", GID: "1000"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume1", Storage: "storage1", Path: "path1", GID: "1000"}, expectedVolumeName: "csi-az-blob-some-component-volume1-storage1", expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-volume1-storage1", }, { name: "Blob CSI Azure volume", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "path1", GID: "1000"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "volume-with-long-name", Storage: "storageName-with-long-name", Path: "path1", GID: "1000"}, expectedVolumeName: "csi-az-blob-some-component-volume-with-long-name-storageN-", expectedVolumeNameIsPrefix: true, expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-volume-with-long-name-storageN-", @@ -275,7 +275,7 @@ func (suite *VolumeMountTestSuite) Test_GetNewVolumes() { suite.T().Run("Unsupported volume type", func(t *testing.T) { t.Parallel() testEnv := getTestEnv() - mounts := []v1.RadixVolumeMount{ + mounts := []radixv1.RadixVolumeMount{ {Type: "unsupported-type", Name: "volume1", Container: "storage1", Path: "path1"}, } component := utils.NewDeployComponentBuilder().WithName(componentName).WithVolumeMounts(mounts...).BuildComponent() @@ -294,7 +294,7 @@ func (suite *VolumeMountTestSuite) Test_GetCsiVolumesWithExistingPvcs() { { volumeMountTestScenario: volumeMountTestScenario{ name: "Blob CSI Azure BlobFuse2 Fuse2 volume", - radixVolumeMount: v1.RadixVolumeMount{Name: "volume1", BlobFuse2: &v1.RadixBlobFuse2VolumeMount{Container: "storage1", GID: "1000"}, Path: "path1"}, + radixVolumeMount: radixv1.RadixVolumeMount{Name: "volume1", BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{Container: "storage1", GID: "1000"}, Path: "path1"}, expectedVolumeName: "csi-blobfuse2-fuse2-some-component-volume1-storage1", }, pvc: createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), @@ -354,20 +354,20 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { { volumeMountTestScenario: volumeMountTestScenario{ name: "Blob CSI Azure volume, Status phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume1", Storage: "storage1", Path: "path1", GID: "1000"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume1", Storage: "storage1", Path: "path1", GID: "1000"}, expectedVolumeName: "csi-az-blob-some-component-blob-volume1-storage1", expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume1-storage1", }, - pvc: createPvc(namespace, componentName, v1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimBound }), + pvc: createPvc(namespace, componentName, radixv1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimBound }), }, { volumeMountTestScenario: volumeMountTestScenario{ name: "Blob CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume2", Storage: "storage2", Path: "path2", GID: "1000"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume2", Storage: "storage2", Path: "path2", GID: "1000"}, expectedVolumeName: "csi-az-blob-some-component-blob-volume2-storage2", expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume2-storage2", }, - pvc: createPvc(namespace, componentName, v1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimPending }), + pvc: createPvc(namespace, componentName, radixv1.MountTypeBlobFuse2FuseCsiAzure, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimPending }), }, } @@ -377,7 +377,7 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { for _, factory := range suite.radixCommonDeployComponentFactories { t.Logf("Test case for component %s", factory.GetTargetType()) - radixDeployment := buildRd(appName, environment, componentName, []v1.RadixVolumeMount{}) + radixDeployment := buildRd(appName, environment, componentName, []radixv1.RadixVolumeMount{}) deployComponent := radixDeployment.Spec.Components[0] volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, radixDeployment.GetNamespace(), &deployComponent, radixDeployment.GetName(), nil) @@ -393,7 +393,7 @@ func (suite *VolumeMountTestSuite) Test_GetVolumesForComponent() { for _, scenario := range scenarios { t.Logf("Test case %s for component %s", scenario.name, factory.GetTargetType()) - radixDeployment := buildRd(appName, environment, componentName, []v1.RadixVolumeMount{scenario.radixVolumeMount}) + radixDeployment := buildRd(appName, environment, componentName, []radixv1.RadixVolumeMount{scenario.radixVolumeMount}) deployComponent := radixDeployment.Spec.Components[0] volumes, err := GetVolumes(context.Background(), testEnv.kubeUtil, radixDeployment.GetNamespace(), &deployComponent, radixDeployment.GetName(), nil) @@ -416,7 +416,7 @@ type expectedPvcPvProperties struct { radixStorageName string pvcName string persistentVolumeName string - radixVolumeMountType v1.MountType + radixVolumeMountType radixv1.MountType requestsVolumeMountSize string volumeAccessMode corev1.PersistentVolumeAccessMode volumeName string @@ -425,6 +425,7 @@ type expectedPvcPvProperties struct { pvGid string pvUid string namespace string + readOnly bool } func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { @@ -434,19 +435,19 @@ func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { scenarios := []volumeMountTestScenario{ { name: "Blob CSI Azure volume, Status phase: Bound", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume1", Storage: "storage1", Path: "path1", GID: "1000"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume1", Storage: "storage1", Path: "path1", GID: "1000"}, expectedVolumeName: "csi-az-blob-some-component-blob-volume1-storage1", expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume1-storage1", }, { name: "Blob CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume2", Storage: "storage2", Path: "path2", GID: "1000"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume2", Storage: "storage2", Path: "path2", GID: "1000"}, expectedVolumeName: "csi-az-blob-some-component-blob-volume2-storage2", expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume2-storage2", }, { name: "Blob CSI Azure volume, Status phase: Pending", - radixVolumeMount: v1.RadixVolumeMount{Type: v1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume-with-long-name", Storage: "storage-with-long-name", Path: "path2", GID: "1000"}, + radixVolumeMount: radixv1.RadixVolumeMount{Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Name: "blob-volume-with-long-name", Storage: "storage-with-long-name", Path: "path2", GID: "1000"}, expectedVolumeName: "csi-az-blob-some-component-blob-volume-with-long-name-sto-", expectedVolumeNameIsPrefix: true, expectedPvcNamePrefix: "pvc-csi-az-blob-some-component-blob-volume-with-long-name-", @@ -458,7 +459,7 @@ func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { for _, factory := range suite.radixCommonDeployComponentFactories { t.Logf("Test case for component %s", factory.GetTargetType()) - radixDeployment := buildRd(appName, environment, componentName, []v1.RadixVolumeMount{}) + radixDeployment := buildRd(appName, environment, componentName, []radixv1.RadixVolumeMount{}) deployComponent := radixDeployment.Spec.Components[0] volumes, err := GetRadixDeployComponentVolumeMounts(&deployComponent, "") @@ -473,7 +474,7 @@ func (suite *VolumeMountTestSuite) Test_GetRadixDeployComponentVolumeMounts() { for _, scenario := range scenarios { t.Logf("Test case %s for component %s", scenario.name, factory.GetTargetType()) - radixDeployment := buildRd(appName, environment, componentName, []v1.RadixVolumeMount{scenario.radixVolumeMount}) + radixDeployment := buildRd(appName, environment, componentName, []radixv1.RadixVolumeMount{scenario.radixVolumeMount}) deployComponent := radixDeployment.Spec.Components[0] volumeMounts, err := GetRadixDeployComponentVolumeMounts(&deployComponent, "") @@ -503,175 +504,175 @@ func modifyPv(pv corev1.PersistentVolume, modify func(pv *corev1.PersistentVolum func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { var scenarios []deploymentVolumesTestScenario - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // return deploymentVolumesTestScenario{ - // name: "Create new volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, nil), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{}, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), - // }, - // existingPvs: []corev1.PersistentVolume{}, - // expectedPvs: []corev1.PersistentVolume{ - // createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // type scenarioProperties struct { - // changedNewRadixVolumeName string - // changedNewRadixVolumeStorageName string - // expectedVolumeName string - // expectedNewSecretName string - // expectedNewPvcName string - // expectedNewPvName string - // } - // getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { - // existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) - // existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) - // return deploymentVolumesTestScenario{ - // name: "Update storage in existing volume name and storage", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - // vm.Name = scenarioProps.changedNewRadixVolumeName - // vm.Storage = scenarioProps.changedNewRadixVolumeStorageName - // }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) { - // v.Name = scenarioProps.expectedVolumeName - // }), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{ - // existingPvc, - // }, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // existingPvc, - // createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName - // pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // pvc.Spec.VolumeName = scenarioProps.expectedNewPvName - // }), - // }, - // existingPvs: []corev1.PersistentVolume{ - // existingPv, - // }, - // expectedPvs: []corev1.PersistentVolume{ - // existingPv, - // createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // pv.ObjectMeta.Name = scenarioProps.expectedNewPvName - // pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName - // pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName - // setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) - // pv.Spec.ClaimRef.Name = scenarioProps.expectedNewPvcName - // pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName - // }), - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ - // changedNewRadixVolumeName: "volume101", - // changedNewRadixVolumeStorageName: "storage101", - // expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", - // expectedNewSecretName: "some-component-volume101-csiazurecreds", - // expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", - // expectedNewPvName: "pv-radixvolumemount-some-uuid", - // }), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createRandomPvc(props, props.namespace, props.componentName) - // expectedPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - // }) - // existingPv := createRandomPv(props, props.namespace, props.componentName) - // expectedPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) { - // //pv.Spec.MountOptions = append(pv.Spec.MountOptions, "-o ro") - // }) - // return deploymentVolumesTestScenario{ - // name: "Set readonly volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{ - // existingPvc, - // }, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // existingPvc, - // expectedPvc, - // }, - // existingPvs: []corev1.PersistentVolume{ - // existingPv, - // }, - // expectedPvs: []corev1.PersistentVolume{ - // existingPv, - // expectedPv, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) - //scenarios = append(scenarios, func() []deploymentVolumesTestScenario { - // getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { - // existingPvc := createExpectedPvc(props, nil) - // existingPv := createExpectedPv(props, nil) - // matchPvAndPvc(&existingPv, &existingPvc) - // expectedPv := modifyPv(existingPv, func(pv *corev1.PersistentVolume) { - // pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // pv.Spec.MountOptions = getMountOptions(props, false) - // }) - // expectedPvc := modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { - // pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} - // }) - // return deploymentVolumesTestScenario{ - // name: "Set ReadWriteOnce volume", - // props: props, - // radixVolumeMounts: []v1.RadixVolumeMount{ - // createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), - // }, - // volumes: []corev1.Volume{ - // createTestVolume(props, func(v *corev1.Volume) {}), - // }, - // existingPvcs: []corev1.PersistentVolumeClaim{ - // existingPvc, - // }, - // expectedPvcs: []corev1.PersistentVolumeClaim{ - // existingPvc, - // expectedPvc, - // }, - // existingPvs: []corev1.PersistentVolume{ - // existingPv, - // }, - // expectedPvs: []corev1.PersistentVolume{ - // existingPv, - // expectedPv, - // }, - // } - // } - // return []deploymentVolumesTestScenario{ - // getScenario(getPropsCsiBlobVolume1Storage1(nil)), - // } - //}()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + return deploymentVolumesTestScenario{ + name: "Create new volume", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) {}), + }, + volumes: []corev1.Volume{ + createTestVolume(props, nil), + }, + existingPvcs: []corev1.PersistentVolumeClaim{}, + expectedPvcs: []corev1.PersistentVolumeClaim{ + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}), + }, + existingPvs: []corev1.PersistentVolume{}, + expectedPvs: []corev1.PersistentVolume{ + createExpectedPv(props, func(pv *corev1.PersistentVolume) {}), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + type scenarioProperties struct { + changedNewRadixVolumeName string + changedNewRadixVolumeStorageName string + expectedVolumeName string + expectedNewSecretName string + expectedNewPvcName string + expectedNewPvName string + } + getScenario := func(props expectedPvcPvProperties, scenarioProps scenarioProperties) deploymentVolumesTestScenario { + existingPv := createExpectedPv(props, func(pv *corev1.PersistentVolume) {}) + existingPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) {}) + return deploymentVolumesTestScenario{ + name: "Update storage in existing volume name and storage", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { + vm.Name = scenarioProps.changedNewRadixVolumeName + vm.Storage = scenarioProps.changedNewRadixVolumeStorageName + }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) { + v.Name = scenarioProps.expectedVolumeName + }), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.ObjectMeta.Name = scenarioProps.expectedNewPvcName + pvc.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pvc.Spec.VolumeName = scenarioProps.expectedNewPvName + }), + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + createExpectedPv(props, func(pv *corev1.PersistentVolume) { + pv.ObjectMeta.Name = scenarioProps.expectedNewPvName + pv.ObjectMeta.Labels[kube.RadixVolumeMountNameLabel] = scenarioProps.changedNewRadixVolumeName + pv.ObjectMeta.Annotations[persistentvolume.CsiAnnotationProvisionerDeletionSecretName] = scenarioProps.expectedNewSecretName + setVolumeMountAttribute(pv, props.radixVolumeMountType, scenarioProps.changedNewRadixVolumeStorageName, scenarioProps.expectedNewPvcName) + pv.Spec.ClaimRef.Name = scenarioProps.expectedNewPvcName + pv.Spec.CSI.NodeStageSecretRef.Name = scenarioProps.expectedNewSecretName + }), + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil), scenarioProperties{ + changedNewRadixVolumeName: "volume101", + changedNewRadixVolumeStorageName: "storage101", + expectedVolumeName: "csi-az-blob-some-component-volume101-storage101", + expectedNewSecretName: "some-component-volume101-csiazurecreds", + expectedNewPvcName: "pvc-csi-az-blob-some-component-volume101-storage101-12345", + expectedNewPvName: "pv-radixvolumemount-some-uuid", + }), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createRandomPvc(props, props.namespace, props.componentName) + expectedPvc := createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + }) + existingPv := createRandomPv(props, props.namespace, props.componentName) + expectedPv := createExpectedPv(props, nil) + return deploymentVolumesTestScenario{ + name: "Set readonly volume", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + expectedPvc, + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + expectedPv, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(func(props *expectedPvcPvProperties) { + props.readOnly = false + })), + } + }()...) + scenarios = append(scenarios, func() []deploymentVolumesTestScenario { + getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { + existingPvc := createExpectedPvc(props, nil) + existingPv := createExpectedPv(props, nil) + matchPvAndPvc(&existingPv, &existingPvc) + expectedPv := modifyPv(existingPv, func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + pv.Spec.MountOptions = getMountOptions(props) + }) + expectedPvc := modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + }) + return deploymentVolumesTestScenario{ + name: "Set ReadWriteOnce volume", + props: props, + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteOnce) }), + }, + volumes: []corev1.Volume{ + createTestVolume(props, func(v *corev1.Volume) {}), + }, + existingPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + }, + expectedPvcs: []corev1.PersistentVolumeClaim{ + existingPvc, + expectedPvc, + }, + existingPvs: []corev1.PersistentVolume{ + existingPv, + }, + expectedPvs: []corev1.PersistentVolume{ + existingPv, + expectedPv, + }, + } + } + return []deploymentVolumesTestScenario{ + getScenario(getPropsCsiBlobVolume1Storage1(nil)), + } + }()...) scenarios = append(scenarios, func() []deploymentVolumesTestScenario { getScenario := func(props expectedPvcPvProperties) deploymentVolumesTestScenario { existingPvc := createExpectedPvc(props, nil) @@ -680,8 +681,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { return deploymentVolumesTestScenario{ name: "Set ReadWriteMany volume", props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadWriteMany) }), }, volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), @@ -702,7 +703,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPv, modifyPv(existingPv, func(pv *corev1.PersistentVolume) { pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - pv.Spec.MountOptions = getMountOptions(props, false) + pv.Spec.MountOptions = getMountOptions(modifyProps(props, func(props *expectedPvcPvProperties) { props.readOnly = false })) }), }, } @@ -718,7 +719,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { matchPvAndPvc(&existingPv, &existingPvc) existingPv = modifyPv(existingPv, func(pv *corev1.PersistentVolume) { pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} - pv.Spec.MountOptions = getMountOptions(props, false) + pv.Spec.MountOptions = getMountOptions(modifyProps(props, func(props *expectedPvcPvProperties) { props.readOnly = false })) }) existingPvc = modifyPvc(existingPvc, func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} @@ -726,8 +727,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { return deploymentVolumesTestScenario{ name: "Set ReadOnlyMany volume", props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { vm.AccessMode = string(corev1.ReadOnlyMany) }), }, volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), @@ -748,7 +749,6 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPv, modifyPv(existingPv, func(pv *corev1.PersistentVolume) { pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} - pv.Spec.MountOptions = getMountOptions(props, true) }), }, } @@ -762,8 +762,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { return deploymentVolumesTestScenario{ name: "Create new BlobFuse2 volume has streaming by default and streaming options not set", props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) {}), }, volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), @@ -775,7 +775,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPvs: []corev1.PersistentVolume{}, expectedPvs: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, "--streaming=true", "--use-adls=false") + pv.Spec.MountOptions = getMountOptions(props, "--streaming=true", "--use-adls=false") }), }, } @@ -789,9 +789,9 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { return deploymentVolumesTestScenario{ name: "Create new BlobFuse2 volume has implicit streaming by default and streaming options set", props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &radixv1.RadixVolumeMountStreaming{ StreamCache: pointers.Ptr(uint64(101)), BlockSize: pointers.Ptr(uint64(102)), BufferSize: pointers.Ptr(uint64(103)), @@ -810,7 +810,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPvs: []corev1.PersistentVolume{}, expectedPvs: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, + pv.Spec.MountOptions = getMountOptions(props, "--streaming=true", "--stream-cache-mb=101", "--block-size-mb=102", @@ -832,9 +832,9 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { return deploymentVolumesTestScenario{ name: "Create new BlobFuse2 volume has disabled streaming", props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createBlobFuse2RadixVolumeMount(props, func(vm *v1.RadixVolumeMount) { - vm.BlobFuse2.Streaming = &v1.RadixVolumeMountStreaming{ + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createBlobFuse2RadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) { + vm.BlobFuse2.Streaming = &radixv1.RadixVolumeMountStreaming{ Enabled: pointers.Ptr(false), StreamCache: pointers.Ptr(uint64(101)), BlockSize: pointers.Ptr(uint64(102)), @@ -854,7 +854,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { existingPvs: []corev1.PersistentVolume{}, expectedPvs: []corev1.PersistentVolume{ createExpectedPv(props, func(pv *corev1.PersistentVolume) { - pv.Spec.MountOptions = getMountOptions(props, true, + pv.Spec.MountOptions = getMountOptions(props, "--use-adls=false") }), }, @@ -878,9 +878,9 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { return deploymentVolumesTestScenario{ name: "Do not change existing PersistentVolume with class name, when creating new PVC", props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRandomVolumeMount(func(vm *v1.RadixVolumeMount) { vm.Name = anotherVolumeMountName }), - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRandomVolumeMount(func(vm *radixv1.RadixVolumeMount) { vm.Name = anotherVolumeMountName }), + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) {}), }, volumes: []corev1.Volume{ volume, @@ -916,8 +916,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { return deploymentVolumesTestScenario{ name: "Do not change existing PersistentVolume without class name, when creating new PVC", props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) {}), }, volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), @@ -955,8 +955,8 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { return deploymentVolumesTestScenario{ name: "Do not change existing PVC with class name, when creating new PersistentVolume", props: props, - radixVolumeMounts: []v1.RadixVolumeMount{ - createRadixVolumeMount(props, func(vm *v1.RadixVolumeMount) {}), + radixVolumeMounts: []radixv1.RadixVolumeMount{ + createRadixVolumeMount(props, func(vm *radixv1.RadixVolumeMount) {}), }, volumes: []corev1.Volume{ createTestVolume(props, func(v *corev1.Volume) {}), @@ -984,7 +984,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }()...) suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { - for _, factory := range suite.radixCommonDeployComponentFactories { + for _, factory := range suite.radixCommonDeployComponentFactories[:1] { for _, scenario := range scenarios { t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) testEnv := getTestEnv() @@ -1008,12 +1008,12 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }) } -func createRandomVolumeMount(modify func(mount *v1.RadixVolumeMount)) v1.RadixVolumeMount { - vm := v1.RadixVolumeMount{ +func createRandomVolumeMount(modify func(mount *radixv1.RadixVolumeMount)) radixv1.RadixVolumeMount { + vm := radixv1.RadixVolumeMount{ Name: strings.ToLower(utils.RandString(10)), Path: "/tmp/" + strings.ToLower(utils.RandString(10)), - BlobFuse2: &v1.RadixBlobFuse2VolumeMount{ - Protocol: v1.BlobFuse2ProtocolFuse2, + BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{ + Protocol: radixv1.BlobFuse2ProtocolFuse2, Container: strings.ToLower(utils.RandString(10)), }, } @@ -1025,7 +1025,7 @@ func matchPvAndPvc(pv *corev1.PersistentVolume, pvc *corev1.PersistentVolumeClai pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvc.GetName() pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcNamespace] = pvc.GetNamespace() pv.Spec.ClaimRef = &corev1.ObjectReference{ - APIVersion: "v1", + APIVersion: "radixv1", Kind: k8s.KindPersistentVolumeClaim, Name: pvc.GetName(), Namespace: pvc.GetNamespace(), @@ -1033,6 +1033,11 @@ func matchPvAndPvc(pv *corev1.PersistentVolume, pvc *corev1.PersistentVolumeClai pvc.Spec.VolumeName = pv.Name } +func modifyProps(props expectedPvcPvProperties, modify func(props *expectedPvcPvProperties)) expectedPvcPvProperties { + modify(&props) + return props +} + func createRandomPv(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolume { return createExpectedPv(props, func(pv *corev1.PersistentVolume) { pvName := getCsiAzurePvName() @@ -1045,7 +1050,7 @@ func createRandomPv(props expectedPvcPvProperties, namespace, componentName stri func createRandomPvc(props expectedPvcPvProperties, namespace, componentName string) corev1.PersistentVolumeClaim { return createExpectedPvc(props, func(pvc *corev1.PersistentVolumeClaim) { - pvcName, err := getCsiAzurePvcName(componentName, &v1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: v1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.radixStorageName, Path: "/tmp"}) + pvcName, err := getCsiAzurePvcName(componentName, &radixv1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.radixStorageName, Path: "/tmp"}) if err != nil { panic(err) } @@ -1073,7 +1078,7 @@ func createRandomAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties func createRandomAutoProvisionedPvcWithStorageClass(props expectedPvcPvProperties, namespace, componentName, anotherVolumeMountName string) corev1.PersistentVolumeClaim { return createExpectedAutoProvisionedPvcWithStorageClass(props, func(pvc *corev1.PersistentVolumeClaim) { - pvcName, err := getCsiAzurePvcName(componentName, &v1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: v1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.radixStorageName, Path: "/tmp"}) + pvcName, err := getCsiAzurePvcName(componentName, &radixv1.RadixVolumeMount{Name: props.radixVolumeMountName, Type: radixv1.MountTypeBlobFuse2FuseCsiAzure, Storage: props.radixStorageName, Path: "/tmp"}) if err != nil { panic(err) } @@ -1087,9 +1092,6 @@ func createRandomAutoProvisionedPvcWithStorageClass(props expectedPvcPvPropertie } func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcPvProperties)) expectedPvcPvProperties { - appName := "any-app" - environment := "some-env" - componentName := "some-component" props := expectedPvcPvProperties{ appName: appName, environment: environment, @@ -1099,7 +1101,7 @@ func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcPvProperties)) expec radixStorageName: "storage1", pvcName: "pvc-csi-az-blob-some-component-volume1-storage1-12345", persistentVolumeName: "pv-radixvolumemount-some-uuid", - radixVolumeMountType: v1.MountTypeBlobFuse2FuseCsiAzure, + radixVolumeMountType: radixv1.MountTypeBlobFuse2FuseCsiAzure, requestsVolumeMountSize: "1Mi", volumeAccessMode: corev1.ReadOnlyMany, // default access mode volumeName: "csi-az-blob-some-component-volume1-storage1", @@ -1107,6 +1109,7 @@ func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcPvProperties)) expec pvSecretName: "some-component-volume1-csiazurecreds", pvGid: "1000", pvUid: "", + readOnly: true, } if modify != nil { modify(&props) @@ -1115,9 +1118,6 @@ func getPropsCsiBlobVolume1Storage1(modify func(*expectedPvcPvProperties)) expec } func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcPvProperties)) expectedPvcPvProperties { - appName := "any-app" - environment := "some-env" - componentName := "some-component" props := expectedPvcPvProperties{ appName: appName, environment: environment, @@ -1127,7 +1127,7 @@ func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcPvProperties)) radixStorageName: "storage1", pvcName: "pvc-csi-blobfuse2-fuse2-some-component-volume1-storage1-12345", persistentVolumeName: "pv-radixvolumemount-some-uuid", - radixVolumeMountType: v1.MountTypeBlobFuse2Fuse2CsiAzure, + radixVolumeMountType: radixv1.MountTypeBlobFuse2Fuse2CsiAzure, requestsVolumeMountSize: "1Mi", volumeAccessMode: corev1.ReadOnlyMany, // default access mode volumeName: "csi-blobfuse2-fuse2-some-component-volume1-storage1", @@ -1135,6 +1135,7 @@ func getPropsCsiBlobFuse2Volume1Storage1(modify func(*expectedPvcPvProperties)) pvSecretName: "some-component-volume1-csiazurecreds", pvGid: "1000", pvUid: "", + readOnly: true, } if modify != nil { modify(&props) @@ -1194,7 +1195,7 @@ func getDesiredDeployment(componentName string, volumes []corev1.Volume) *appsv1 } } -func buildRd(appName string, environment string, componentName string, radixVolumeMounts []v1.RadixVolumeMount) *v1.RadixDeployment { +func buildRd(appName string, environment string, componentName string, radixVolumeMounts []radixv1.RadixVolumeMount) *radixv1.RadixDeployment { return utils.ARadixDeployment(). WithAppName(appName). WithEnvironment(environment). @@ -1204,7 +1205,7 @@ func buildRd(appName string, environment string, componentName string, radixVolu BuildRD() } -func createPvc(namespace, componentName string, mountType v1.MountType, modify func(*corev1.PersistentVolumeClaim)) corev1.PersistentVolumeClaim { +func createPvc(namespace, componentName string, mountType radixv1.MountType, modify func(*corev1.PersistentVolumeClaim)) corev1.PersistentVolumeClaim { appName := "app" pvc := corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -1225,7 +1226,7 @@ func createPvc(namespace, componentName string, mountType v1.MountType, modify f } func createExpectedPv(props expectedPvcPvProperties, modify func(pv *corev1.PersistentVolume)) corev1.PersistentVolume { - mountOptions := getMountOptions(props, true) + mountOptions := getMountOptions(props) pv := corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: props.persistentVolumeName, @@ -1264,7 +1265,7 @@ func createExpectedPv(props expectedPvcPvProperties, modify func(pv *corev1.Pers }, AccessModes: []corev1.PersistentVolumeAccessMode{props.volumeAccessMode}, ClaimRef: &corev1.ObjectReference{ - APIVersion: "v1", + APIVersion: "radixv1", Kind: k8s.KindPersistentVolumeClaim, Namespace: props.namespace, Name: props.pvcName, @@ -1284,7 +1285,7 @@ func createExpectedPv(props expectedPvcPvProperties, modify func(pv *corev1.Pers } func createAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties, modify func(pv *corev1.PersistentVolume)) corev1.PersistentVolume { - mountOptions := getMountOptionsInRandomOrder(props, true) + mountOptions := getMountOptionsInRandomOrder(props) pv := corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: props.persistentVolumeName, @@ -1317,7 +1318,7 @@ func createAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties, modi }, AccessModes: []corev1.PersistentVolumeAccessMode{props.volumeAccessMode}, ClaimRef: &corev1.ObjectReference{ - APIVersion: "v1", + APIVersion: "radixv1", Kind: k8s.KindPersistentVolumeClaim, Namespace: props.namespace, Name: props.pvcName, @@ -1336,7 +1337,7 @@ func createAutoProvisionedPvWithStorageClass(props expectedPvcPvProperties, modi return pv } -func getMountOptions(props expectedPvcPvProperties, readOnly bool, extraOptions ...string) []string { +func getMountOptions(props expectedPvcPvProperties, extraOptions ...string) []string { options := []string{ "--file-cache-timeout-in-seconds=120", "--use-attr-cache=true", @@ -1346,7 +1347,7 @@ func getMountOptions(props expectedPvcPvProperties, readOnly bool, extraOptions "-o entry_timeout=120", "-o negative_timeout=120", } - if readOnly { + if props.readOnly { options = append(options, ReadOnlyMountOption) } idOption := getPersistentVolumeIdMountOption(props) @@ -1356,7 +1357,7 @@ func getMountOptions(props expectedPvcPvProperties, readOnly bool, extraOptions return append(options, extraOptions...) } -func getMountOptionsInRandomOrder(props expectedPvcPvProperties, readOnly bool, extraOptions ...string) []string { +func getMountOptionsInRandomOrder(props expectedPvcPvProperties, extraOptions ...string) []string { options := []string{ "--file-cache-timeout-in-seconds=120", "--use-attr-cache=true", @@ -1370,19 +1371,19 @@ func getMountOptionsInRandomOrder(props expectedPvcPvProperties, readOnly bool, if len(idOption) > 0 { options = append(options, idOption) } - if readOnly { + if props.readOnly { options = append(options, ReadOnlyMountOption) } return append(options, extraOptions...) } -func setVolumeMountAttribute(pv *corev1.PersistentVolume, radixVolumeMountType v1.MountType, containerName, pvcName string) { +func setVolumeMountAttribute(pv *corev1.PersistentVolume, radixVolumeMountType radixv1.MountType, containerName, pvcName string) { pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributeContainerName] = containerName pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributePvcName] = pvcName switch radixVolumeMountType { - case v1.MountTypeBlobFuse2FuseCsiAzure: + case radixv1.MountTypeBlobFuse2FuseCsiAzure: pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributeProtocol] = persistentvolume.CsiVolumeAttributeProtocolParameterFuse - case v1.MountTypeBlobFuse2Fuse2CsiAzure: + case radixv1.MountTypeBlobFuse2Fuse2CsiAzure: pv.Spec.CSI.VolumeAttributes[persistentvolume.CsiVolumeMountAttributeProtocol] = persistentvolume.CsiVolumeAttributeProtocolParameterFuse2 } } @@ -1495,8 +1496,8 @@ func createRandomVolume(modify func(*corev1.Volume)) corev1.Volume { return volume } -func createRadixVolumeMount(props expectedPvcPvProperties, modify func(mount *v1.RadixVolumeMount)) v1.RadixVolumeMount { - volumeMount := v1.RadixVolumeMount{ +func createRadixVolumeMount(props expectedPvcPvProperties, modify func(mount *radixv1.RadixVolumeMount)) radixv1.RadixVolumeMount { + volumeMount := radixv1.RadixVolumeMount{ Type: props.radixVolumeMountType, Name: props.radixVolumeMountName, Storage: props.radixStorageName, @@ -1508,11 +1509,11 @@ func createRadixVolumeMount(props expectedPvcPvProperties, modify func(mount *v1 } return volumeMount } -func createBlobFuse2RadixVolumeMount(props expectedPvcPvProperties, modify func(mount *v1.RadixVolumeMount)) v1.RadixVolumeMount { - volumeMount := v1.RadixVolumeMount{ +func createBlobFuse2RadixVolumeMount(props expectedPvcPvProperties, modify func(mount *radixv1.RadixVolumeMount)) radixv1.RadixVolumeMount { + volumeMount := radixv1.RadixVolumeMount{ Name: props.radixVolumeMountName, Path: "path1", - BlobFuse2: &v1.RadixBlobFuse2VolumeMount{ + BlobFuse2: &radixv1.RadixBlobFuse2VolumeMount{ Container: props.radixStorageName, GID: "1000", }, From d07121ee5b7aebe00a0c97f9ad2c705f29711f0e Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 14 Jan 2025 10:41:02 +0100 Subject: [PATCH 63/68] Cleaned outdated code --- pkg/apis/volumemount/volumemount.go | 4 ---- pkg/apis/volumemount/volumemount_test.go | 15 +-------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index a8aa17cb2..f553ffaa9 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -469,10 +469,6 @@ func getCsiAzurePvcs(ctx context.Context, kubeClient kubernetes.Interface, names return kubeClient.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) } -func getLabelSelectorForCsiAzurePvc(componentName string) string { - return fmt.Sprintf("%s=%s, %s in (%s, %s)", kube.RadixComponentLabel, componentName, kube.RadixMountTypeLabel, string(radixv1.MountTypeBlobFuse2FuseCsiAzure), string(radixv1.MountTypeBlobFuse2Fuse2CsiAzure)) -} - func buildPvc(appName, namespace, componentName, pvName, pvcName string, radixVolumeMount *radixv1.RadixVolumeMount) *corev1.PersistentVolumeClaim { return &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index a923f5812..78ff00692 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -84,7 +84,6 @@ const ( ) var ( - anotherNamespace = strings.ToLower(commonUtils.RandString(10)) anotherComponentName = strings.ToLower(commonUtils.RandString(10)) anotherVolumeMountName = strings.ToLower(commonUtils.RandString(10)) ) @@ -1483,19 +1482,6 @@ func createTestVolume(pvcProps expectedPvcPvProperties, modify func(v *corev1.Vo return volume } -func createRandomVolume(modify func(*corev1.Volume)) corev1.Volume { - volume := corev1.Volume{ - Name: strings.ToLower(utils.RandString(10)), - VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: strings.ToLower(utils.RandString(10)), - }}, - } - if modify != nil { - modify(&volume) - } - return volume -} - func createRadixVolumeMount(props expectedPvcPvProperties, modify func(mount *radixv1.RadixVolumeMount)) radixv1.RadixVolumeMount { volumeMount := radixv1.RadixVolumeMount{ Type: props.radixVolumeMountType, @@ -1509,6 +1495,7 @@ func createRadixVolumeMount(props expectedPvcPvProperties, modify func(mount *ra } return volumeMount } + func createBlobFuse2RadixVolumeMount(props expectedPvcPvProperties, modify func(mount *radixv1.RadixVolumeMount)) radixv1.RadixVolumeMount { volumeMount := radixv1.RadixVolumeMount{ Name: props.radixVolumeMountName, From 29f80a726f10adcf3e3fbc951487d829909f129e Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 14 Jan 2025 12:35:14 +0100 Subject: [PATCH 64/68] Cleaned outdated code --- pkg/apis/deployment/deployment.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/apis/deployment/deployment.go b/pkg/apis/deployment/deployment.go index 600d49160..b7a842170 100644 --- a/pkg/apis/deployment/deployment.go +++ b/pkg/apis/deployment/deployment.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/equinor/radix-operator/pkg/apis/volumemount" "sort" "strings" "time" @@ -20,6 +19,7 @@ import ( "github.com/equinor/radix-operator/pkg/apis/metrics" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/volumemount" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" monitoring "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" "github.com/rs/zerolog/log" @@ -633,7 +633,5 @@ func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(ctx context.Conte } return nil, nil, err } - desiredJobAuxDeployment := currentJobAuxDeployment.DeepCopy() - desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].Resources = getJobAuxResources() - return currentJobAuxDeployment, desiredJobAuxDeployment, nil + return currentJobAuxDeployment, currentJobAuxDeployment.DeepCopy(), nil } From dcbb528daee236bea6077150414d9fc2771d897a Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 14 Jan 2025 12:48:42 +0100 Subject: [PATCH 65/68] Cleanup --- pkg/apis/deployment/kubedeployment.go | 4 ++-- pkg/apis/deployment/secrets.go | 4 ++-- pkg/apis/deployment/service.go | 4 ++-- pkg/apis/deployment/serviceaccountspec.go | 2 +- pkg/apis/deployment/serviceaccountspec_test.go | 4 ++-- pkg/apis/deployment/volume_test.go | 7 ++++--- pkg/apis/metrics/custom_metrics.go | 2 +- pkg/apis/test/utils.go | 2 +- pkg/apis/utils/hash/hash.go | 9 ++++----- pkg/apis/utils/hash/hash_test.go | 10 +++++----- pkg/apis/volumemount/volumemount.go | 1 - 11 files changed, 24 insertions(+), 25 deletions(-) diff --git a/pkg/apis/deployment/kubedeployment.go b/pkg/apis/deployment/kubedeployment.go index fe76854d7..840831349 100644 --- a/pkg/apis/deployment/kubedeployment.go +++ b/pkg/apis/deployment/kubedeployment.go @@ -2,18 +2,18 @@ package deployment import ( "context" - internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" - "github.com/equinor/radix-operator/pkg/apis/volumemount" commonUtils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/defaults" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "github.com/equinor/radix-operator/pkg/apis/kube" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/securitycontext" "github.com/equinor/radix-operator/pkg/apis/utils" radixannotations "github.com/equinor/radix-operator/pkg/apis/utils/annotations" radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" + "github.com/equinor/radix-operator/pkg/apis/volumemount" "github.com/rs/zerolog/log" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" diff --git a/pkg/apis/deployment/secrets.go b/pkg/apis/deployment/secrets.go index 1c324f83c..8fd08df17 100644 --- a/pkg/apis/deployment/secrets.go +++ b/pkg/apis/deployment/secrets.go @@ -3,7 +3,6 @@ package deployment import ( "context" "fmt" - "github.com/equinor/radix-operator/pkg/apis/volumemount" "strings" "github.com/equinor/radix-common/utils/slice" @@ -12,8 +11,9 @@ import ( "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/apis/volumemount" "github.com/rs/zerolog/log" - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/pkg/apis/deployment/service.go b/pkg/apis/deployment/service.go index 562b35bd0..f6f5b2ba3 100644 --- a/pkg/apis/deployment/service.go +++ b/pkg/apis/deployment/service.go @@ -2,10 +2,10 @@ package deployment import ( "context" - internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "github.com/equinor/radix-operator/pkg/apis/kube" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/apis/deployment/serviceaccountspec.go b/pkg/apis/deployment/serviceaccountspec.go index b63deee63..4e1cf643f 100644 --- a/pkg/apis/deployment/serviceaccountspec.go +++ b/pkg/apis/deployment/serviceaccountspec.go @@ -4,7 +4,7 @@ import ( "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/defaults" internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" ) diff --git a/pkg/apis/deployment/serviceaccountspec_test.go b/pkg/apis/deployment/serviceaccountspec_test.go index 3d5cc1543..f41a48c79 100644 --- a/pkg/apis/deployment/serviceaccountspec_test.go +++ b/pkg/apis/deployment/serviceaccountspec_test.go @@ -1,12 +1,12 @@ package deployment import ( - internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" "testing" "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-operator/pkg/apis/defaults" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + internal "github.com/equinor/radix-operator/pkg/apis/internal/deployment" + "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/stretchr/testify/assert" ) diff --git a/pkg/apis/deployment/volume_test.go b/pkg/apis/deployment/volume_test.go index d3871abf6..a67544868 100644 --- a/pkg/apis/deployment/volume_test.go +++ b/pkg/apis/deployment/volume_test.go @@ -2,13 +2,14 @@ package deployment import ( "context" + "testing" + "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/api/resource" - v2 "k8s.io/apimachinery/pkg/apis/meta/v1" - "testing" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func Test_EmptyDir(t *testing.T) { @@ -30,7 +31,7 @@ func Test_EmptyDir(t *testing.T) { require.NoError(t, err) assert.NotNil(t, rd) - deployment, err := kubeclient.AppsV1().Deployments(utils.GetEnvironmentNamespace(appName, envName)).Get(context.Background(), compName, v2.GetOptions{}) + deployment, err := kubeclient.AppsV1().Deployments(utils.GetEnvironmentNamespace(appName, envName)).Get(context.Background(), compName, metav1.GetOptions{}) require.NoError(t, err) assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 2) assert.Len(t, deployment.Spec.Template.Spec.Volumes, 2) diff --git a/pkg/apis/metrics/custom_metrics.go b/pkg/apis/metrics/custom_metrics.go index 6869b7659..c5aca2613 100644 --- a/pkg/apis/metrics/custom_metrics.go +++ b/pkg/apis/metrics/custom_metrics.go @@ -6,7 +6,7 @@ import ( "time" "github.com/equinor/radix-operator/pkg/apis/defaults" - v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" diff --git a/pkg/apis/test/utils.go b/pkg/apis/test/utils.go index 703ef2898..4c9a55e21 100644 --- a/pkg/apis/test/utils.go +++ b/pkg/apis/test/utils.go @@ -3,10 +3,10 @@ package test import ( "context" "errors" - commonUtils "github.com/equinor/radix-common/utils" "os" "time" + commonUtils "github.com/equinor/radix-common/utils" "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" diff --git a/pkg/apis/utils/hash/hash.go b/pkg/apis/utils/hash/hash.go index 96983a848..5ffd9ba2a 100644 --- a/pkg/apis/utils/hash/hash.go +++ b/pkg/apis/utils/hash/hash.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/equinor/radix-operator/pkg/apis/radix/v1" - v2 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" ) type Algorithm string @@ -68,7 +68,6 @@ func extractAlgorithmFromHashString(hashString string) Algorithm { const ( magicValueForNilRadixApplication = "0nXSg9l6EUepshGFmolpgV3elB0m8Mv7" magicValueForNilBuildSecretData = "34Wd68DsJRUzrHp2f63o3U5hUD6zl8Tj" - magicValueForNilVolumeMountData = "50gfDfg6h65fGfdVfs4r5dhnGFF4D35f" ) func CreateRadixApplicationHash(ra *v1.RadixApplication) (string, error) { @@ -79,11 +78,11 @@ func CompareRadixApplicationHash(targetHash string, ra *v1.RadixApplication) (bo return CompareWithHashString(getRadixApplicationOrMagicValue(ra), targetHash) } -func CreateBuildSecretHash(secret *v2.Secret) (string, error) { +func CreateBuildSecretHash(secret *corev1.Secret) (string, error) { return ToHashString(SHA256, getBuildSecretOrMagicValue(secret)) } -func CompareBuildSecretHash(targetHash string, secret *v2.Secret) (bool, error) { +func CompareBuildSecretHash(targetHash string, secret *corev1.Secret) (bool, error) { return CompareWithHashString(getBuildSecretOrMagicValue(secret), targetHash) } @@ -94,7 +93,7 @@ func getRadixApplicationOrMagicValue(ra *v1.RadixApplication) any { return ra.Spec } -func getBuildSecretOrMagicValue(secret *v2.Secret) any { +func getBuildSecretOrMagicValue(secret *corev1.Secret) any { if secret == nil || len(secret.Data) == 0 { return magicValueForNilBuildSecretData } diff --git a/pkg/apis/utils/hash/hash_test.go b/pkg/apis/utils/hash/hash_test.go index b21b18f9d..2d0fadf1c 100644 --- a/pkg/apis/utils/hash/hash_test.go +++ b/pkg/apis/utils/hash/hash_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - hash2 "github.com/equinor/radix-operator/pkg/apis/utils/hash" + "github.com/equinor/radix-operator/pkg/apis/utils/hash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,10 +16,10 @@ func Test_ToHashString(t *testing.T) { } type algorithm struct { - alg hash2.Algorithm + alg hash.Algorithm hashLen int } - algorithms := []algorithm{{alg: hash2.SHA256, hashLen: 32}} + algorithms := []algorithm{{alg: hash.SHA256, hashLen: 32}} type testSpec struct { test string @@ -65,14 +65,14 @@ func Test_ToHashString(t *testing.T) { for _, test := range tests { for _, alg := range algorithms { t.Run(test.test, func(t *testing.T) { - valHash, err := hash2.ToHashString(alg.alg, test.val) + valHash, err := hash.ToHashString(alg.alg, test.val) require.NoError(t, err) hashParts := strings.Split(valHash, "=") assert.Equal(t, string(alg.alg), hashParts[0]) valHashBytes, err := hex.DecodeString(hashParts[1]) require.NoError(t, err) assert.Len(t, valHashBytes, alg.hashLen) - match, err := hash2.CompareWithHashString(test.valOther, valHash) + match, err := hash.CompareWithHashString(test.valOther, valHash) require.NoError(t, err) assert.Equal(t, test.match, match) t.Parallel() diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index f553ffaa9..341c4a701 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -91,7 +91,6 @@ func GarbageCollectVolumeMountsSecretsNoLongerInSpecForComponent(ctx context.Con return err } } - return garbageCollectSecrets(ctx, kubeUtil, namespace, secrets, excludeSecretNames) } From 40a51e2e0a23afaeac9249acb4cb3b828044b06f Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 14 Jan 2025 12:56:41 +0100 Subject: [PATCH 66/68] Cleanup --- pkg/apis/volumemount/volumemount.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index 341c4a701..fdf61974e 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -911,7 +911,7 @@ func appendPvcNamesFromVolumes(pvcMap map[string]any, volumes []corev1.Volume) m } func garbageCollectCsiAzurePvs(ctx context.Context, kubeClient kubernetes.Interface, namespace string, excludePvcNames map[string]any) error { - pvs, err := getCsiAzurePvsForNamespace(ctx, kubeClient, namespace, true) + pvs, err := getCsiAzurePvsForNamespace(ctx, kubeClient, namespace, false) if err != nil { return err } From fe6adaa7771d4ab87550c722bd359ae31a715336 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Wed, 22 Jan 2025 12:53:50 +0100 Subject: [PATCH 67/68] Corrected comments. Deleted not supported AzureFile type and property --- .../templates/radixapplication.yaml | 271 +++--------------- .../templates/radixdeployment.yaml | 132 ++------- json-schema/radixapplication.json | 258 +++-------------- .../persistentvolume/persistentvolume.go | 1 - pkg/apis/radix/v1/radixapptypes.go | 68 +---- pkg/apis/radix/v1/zz_generated.deepcopy.go | 21 -- pkg/apis/radixvalidators/errors.go | 57 ++-- .../utils/applicationcomponent_builder.go | 48 ++-- 8 files changed, 159 insertions(+), 697 deletions(-) diff --git a/charts/radix-operator/templates/radixapplication.yaml b/charts/radix-operator/templates/radixapplication.yaml index 23e4c77f8..326b1d8d0 100644 --- a/charts/radix-operator/templates/radixapplication.yaml +++ b/charts/radix-operator/templates/radixapplication.yaml @@ -1369,69 +1369,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: |- - AzureFile settings for Azure File CSI driver - Deprecated. - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) - which will be set as owner of the mounted - volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in - the external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) - which will be set as owner of the mounted - volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -1559,7 +1510,7 @@ spec: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -1578,8 +1529,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -1595,32 +1546,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 instead. enum: - - blob - azure-blob - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string useAzureIdentity: description: UseAzureIdentity defines that credentials @@ -2451,7 +2401,8 @@ spec: - name x-kubernetes-list-type: map public: - description: Deprecated, use publicPort instead. + description: 'Deprecated: For backwards compatibility Public + is still supported, new code should use PublicPort instead' type: boolean publicPort: description: |- @@ -2623,67 +2574,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: |- - AzureFile settings for Azure File CSI driver - Deprecated. - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) which - will be set as owner of the mounted volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in the - external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) which - will be set as owner of the mounted volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -2806,7 +2710,7 @@ spec: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -2825,8 +2729,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -2842,32 +2746,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 instead. enum: - - blob - azure-blob - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string useAzureIdentity: description: UseAzureIdentity defines that credentials @@ -3560,69 +3463,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: |- - AzureFile settings for Azure File CSI driver - Deprecated. - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) - which will be set as owner of the mounted - volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in - the external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) - which will be set as owner of the mounted - volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -3750,7 +3604,7 @@ spec: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -3769,8 +3623,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -3786,32 +3640,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 instead. enum: - - blob - azure-blob - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string useAzureIdentity: description: UseAzureIdentity defines that credentials @@ -4195,67 +4048,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: |- - AzureFile settings for Azure File CSI driver - Deprecated. - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) which - will be set as owner of the mounted volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in the - external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) which - will be set as owner of the mounted volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -4378,7 +4184,7 @@ spec: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -4397,8 +4203,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -4414,32 +4220,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 instead. enum: - - blob - azure-blob - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string useAzureIdentity: description: UseAzureIdentity defines that credentials diff --git a/charts/radix-operator/templates/radixdeployment.yaml b/charts/radix-operator/templates/radixdeployment.yaml index 979168b69..97456aed9 100644 --- a/charts/radix-operator/templates/radixdeployment.yaml +++ b/charts/radix-operator/templates/radixdeployment.yaml @@ -1171,67 +1171,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: |- - AzureFile settings for Azure File CSI driver - Deprecated. - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) which - will be set as owner of the mounted volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in the - external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) which - will be set as owner of the mounted volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -1354,7 +1307,7 @@ spec: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -1373,8 +1326,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -1390,32 +1343,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 instead. enum: - - blob - azure-blob - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string useAzureIdentity: description: UseAzureIdentity defines that credentials @@ -1834,67 +1786,20 @@ spec: properties: accessMode: description: |- + Deprecated: use BlobFuse2 instead. Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - ReadOnlyMany - ReadWriteOnce - ReadWriteMany - "" type: string - azureFile: - description: |- - AzureFile settings for Azure File CSI driver - Deprecated. - properties: - accessMode: - description: |- - Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - ReadOnlyMany - - ReadWriteOnce - - ReadWriteMany - - "" - type: string - bindingMode: - description: |- - Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - enum: - - Immediate - - WaitForFirstConsumer - - "" - type: string - gid: - description: GID defines the group ID (number) which - will be set as owner of the mounted volume. - type: string - requestsStorage: - description: |- - Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - type: string - share: - description: Share. Name of the file share in the - external storage resource. - type: string - skuName: - description: |- - SKU Type of Azure storage. - More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - type: string - uid: - description: UID defines the user ID (number) which - will be set as owner of the mounted volume. - type: string - type: object bindingMode: description: |- + Deprecated: use BlobFuse2 instead. Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. enum: - Immediate - WaitForFirstConsumer @@ -2017,7 +1922,7 @@ spec: - container type: object container: - description: 'Deprecated. Only required by the deprecated + description: 'Deprecated: Only required by the deprecated type: blob.' type: string emptyDir: @@ -2036,8 +1941,8 @@ spec: type: object gid: description: |- + Deprecated: use BlobFuse2 instead. GID defines the group ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string name: description: |- @@ -2053,32 +1958,31 @@ spec: type: string requestsStorage: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string skuName: description: |- + Deprecated: use BlobFuse2 instead. More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - Deprecated, use BlobFuse2 instead. type: string storage: description: |- + Deprecated: use BlobFuse2 instead. Storage defines the name of the container in the external storage resource. - Deprecated, use BlobFuse2 instead. type: string type: description: |- + Deprecated: use BlobFuse2 instead. Type defines the storage type. - Deprecated, use BlobFuse2 instead. enum: - - blob - azure-blob - "" type: string uid: description: |- + Deprecated: use BlobFuse2 instead. UID defines the user ID (number) which will be set as owner of the mounted volume. - Deprecated, use BlobFuse2 instead. type: string useAzureIdentity: description: UseAzureIdentity defines that credentials diff --git a/json-schema/radixapplication.json b/json-schema/radixapplication.json index e390365b7..ab7b0d588 100644 --- a/json-schema/radixapplication.json +++ b/json-schema/radixapplication.json @@ -1335,7 +1335,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nAccess mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -1344,53 +1344,8 @@ ], "type": "string" }, - "azureFile": { - "description": "AzureFile settings for Azure File CSI driver\nDeprecated.", - "properties": { - "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "ReadOnlyMany", - "ReadWriteOnce", - "ReadWriteMany", - "" - ], - "type": "string" - }, - "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "Immediate", - "WaitForFirstConsumer", - "" - ], - "type": "string" - }, - "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.", - "type": "string" - }, - "requestsStorage": { - "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", - "type": "string" - }, - "share": { - "description": "Share. Name of the file share in the external storage resource.", - "type": "string" - }, - "skuName": { - "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", - "type": "string" - }, - "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", - "type": "string" - } - }, - "type": "object" - }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nBinding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -1514,7 +1469,7 @@ "type": "object" }, "container": { - "description": "Deprecated. Only required by the deprecated type: blob.", + "description": "Deprecated: Only required by the deprecated type: blob.", "type": "string" }, "emptyDir": { @@ -1540,7 +1495,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nGID defines the group ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "name": { @@ -1555,28 +1510,27 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nStorage defines the name of the container in the external storage resource.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nType defines the storage type.", "enum": [ - "blob", "azure-blob", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nUID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "useAzureIdentity": { @@ -2441,7 +2395,7 @@ "x-kubernetes-list-type": "map" }, "public": { - "description": "Deprecated, use publicPort instead.", + "description": "Deprecated: For backwards compatibility Public is still supported, new code should use PublicPort instead", "type": "boolean" }, "publicPort": { @@ -2613,7 +2567,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nAccess mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -2622,53 +2576,8 @@ ], "type": "string" }, - "azureFile": { - "description": "AzureFile settings for Azure File CSI driver\nDeprecated.", - "properties": { - "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "ReadOnlyMany", - "ReadWriteOnce", - "ReadWriteMany", - "" - ], - "type": "string" - }, - "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "Immediate", - "WaitForFirstConsumer", - "" - ], - "type": "string" - }, - "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.", - "type": "string" - }, - "requestsStorage": { - "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", - "type": "string" - }, - "share": { - "description": "Share. Name of the file share in the external storage resource.", - "type": "string" - }, - "skuName": { - "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", - "type": "string" - }, - "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", - "type": "string" - } - }, - "type": "object" - }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nBinding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -2792,7 +2701,7 @@ "type": "object" }, "container": { - "description": "Deprecated. Only required by the deprecated type: blob.", + "description": "Deprecated: Only required by the deprecated type: blob.", "type": "string" }, "emptyDir": { @@ -2818,7 +2727,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nGID defines the group ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "name": { @@ -2833,28 +2742,27 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nStorage defines the name of the container in the external storage resource.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nType defines the storage type.", "enum": [ - "blob", "azure-blob", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nUID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "useAzureIdentity": { @@ -3578,7 +3486,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nAccess mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -3587,53 +3495,8 @@ ], "type": "string" }, - "azureFile": { - "description": "AzureFile settings for Azure File CSI driver\nDeprecated.", - "properties": { - "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "ReadOnlyMany", - "ReadWriteOnce", - "ReadWriteMany", - "" - ], - "type": "string" - }, - "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "Immediate", - "WaitForFirstConsumer", - "" - ], - "type": "string" - }, - "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.", - "type": "string" - }, - "requestsStorage": { - "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", - "type": "string" - }, - "share": { - "description": "Share. Name of the file share in the external storage resource.", - "type": "string" - }, - "skuName": { - "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", - "type": "string" - }, - "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", - "type": "string" - } - }, - "type": "object" - }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nBinding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -3757,7 +3620,7 @@ "type": "object" }, "container": { - "description": "Deprecated. Only required by the deprecated type: blob.", + "description": "Deprecated: Only required by the deprecated type: blob.", "type": "string" }, "emptyDir": { @@ -3783,7 +3646,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nGID defines the group ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "name": { @@ -3798,28 +3661,27 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nStorage defines the name of the container in the external storage resource.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nType defines the storage type.", "enum": [ - "blob", "azure-blob", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nUID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "useAzureIdentity": { @@ -4212,7 +4074,7 @@ "description": "RadixVolumeMount defines an external storage resource.", "properties": { "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nAccess mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "ReadOnlyMany", "ReadWriteOnce", @@ -4221,53 +4083,8 @@ ], "type": "string" }, - "azureFile": { - "description": "AzureFile settings for Azure File CSI driver\nDeprecated.", - "properties": { - "accessMode": { - "description": "Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "ReadOnlyMany", - "ReadWriteOnce", - "ReadWriteMany", - "" - ], - "type": "string" - }, - "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", - "enum": [ - "Immediate", - "WaitForFirstConsumer", - "" - ], - "type": "string" - }, - "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.", - "type": "string" - }, - "requestsStorage": { - "description": "Requested size (opens new window)of allocated mounted volume. Default value is set to \"1Mi\" (1 megabyte). Current version of the driver does not affect mounted volume size\nMore info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim", - "type": "string" - }, - "share": { - "description": "Share. Name of the file share in the external storage resource.", - "type": "string" - }, - "skuName": { - "description": "SKU Type of Azure storage.\nMore info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types", - "type": "string" - }, - "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.", - "type": "string" - } - }, - "type": "object" - }, "bindingMode": { - "description": "Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nBinding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "enum": [ "Immediate", "WaitForFirstConsumer", @@ -4391,7 +4208,7 @@ "type": "object" }, "container": { - "description": "Deprecated. Only required by the deprecated type: blob.", + "description": "Deprecated: Only required by the deprecated type: blob.", "type": "string" }, "emptyDir": { @@ -4417,7 +4234,7 @@ "type": "object" }, "gid": { - "description": "GID defines the group ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nGID defines the group ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "name": { @@ -4432,28 +4249,27 @@ "type": "string" }, "requestsStorage": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "skuName": { - "description": "More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nMore info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/", "type": "string" }, "storage": { - "description": "Storage defines the name of the container in the external storage resource.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nStorage defines the name of the container in the external storage resource.", "type": "string" }, "type": { - "description": "Type defines the storage type.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nType defines the storage type.", "enum": [ - "blob", "azure-blob", "" ], "type": "string" }, "uid": { - "description": "UID defines the user ID (number) which will be set as owner of the mounted volume.\nDeprecated, use BlobFuse2 instead.", + "description": "Deprecated: use BlobFuse2 instead.\nUID defines the user ID (number) which will be set as owner of the mounted volume.", "type": "string" }, "useAzureIdentity": { diff --git a/pkg/apis/internal/persistentvolume/persistentvolume.go b/pkg/apis/internal/persistentvolume/persistentvolume.go index 1e273bc86..a16af88f0 100644 --- a/pkg/apis/internal/persistentvolume/persistentvolume.go +++ b/pkg/apis/internal/persistentvolume/persistentvolume.go @@ -30,7 +30,6 @@ func EqualPersistentVolumes(pv1, pv2 *corev1.PersistentVolume) bool { return false } - // ignore pv1.Spec.StorageClassName != pv2.Spec.StorageClassName for transition period. When there is no volumeMounts with storage class exists - uncomment this line if pv1.Spec.Capacity[corev1.ResourceStorage] != pv2.Spec.Capacity[corev1.ResourceStorage] || len(pv1.Spec.AccessModes) != len(pv2.Spec.AccessModes) || (len(pv1.Spec.AccessModes) != 1 && pv1.Spec.AccessModes[0] != pv2.Spec.AccessModes[0]) || diff --git a/pkg/apis/radix/v1/radixapptypes.go b/pkg/apis/radix/v1/radixapptypes.go index c3e21d6c2..3ee829355 100644 --- a/pkg/apis/radix/v1/radixapptypes.go +++ b/pkg/apis/radix/v1/radixapptypes.go @@ -380,9 +380,9 @@ type RadixComponent struct { // +optional Monitoring *bool `json:"monitoring"` - // Deprecated, use publicPort instead. + // Deprecated: For backwards compatibility Public is still supported, new code should use PublicPort instead // +optional - Public bool `json:"public,omitempty"` // Deprecated: For backwards compatibility Public is still supported, new code should use PublicPort instead + Public bool `json:"public,omitempty"` // Defines which port (name) from the ports list that shall be accessible from the internet. // More info: https://www.radix.equinor.com/references/reference-radix-config/#publicport @@ -919,9 +919,9 @@ type RadixPrivateImageHubCredential struct { // RadixVolumeMount defines an external storage resource. type RadixVolumeMount struct { + // Deprecated: use BlobFuse2 instead. // Type defines the storage type. - // Deprecated, use BlobFuse2 instead. - // +kubebuilder:validation:Enum=blob;azure-blob;"" + // +kubebuilder:validation:Enum=azure-blob;"" // +optional Type MountType `json:"type"` @@ -931,12 +931,12 @@ type RadixVolumeMount struct { // +kubebuilder:validation:MaxLength=40 Name string `json:"name"` - // Deprecated. Only required by the deprecated type: blob. + // Deprecated: Only required by the deprecated type: blob. // +optional Container string `json:"container,omitempty"` // Outdated. Use Storage instead + // Deprecated: use BlobFuse2 instead. // Storage defines the name of the container in the external storage resource. - // Deprecated, use BlobFuse2 instead. // +optional Storage string `json:"storage"` // Container name, file Share name, etc. @@ -944,38 +944,36 @@ type RadixVolumeMount struct { // +kubebuilder:validation:MinLength=1 Path string `json:"path"` // Path within the pod (replica), where the volume mount has been mounted to + // Deprecated: use BlobFuse2 instead. // GID defines the group ID (number) which will be set as owner of the mounted volume. - // Deprecated, use BlobFuse2 instead. // +optional GID string `json:"gid,omitempty"` // Optional. Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting. https://github.com/kubernetes-sigs/blob-csi-driver/blob/master/docs/driver-parameters.md + // Deprecated: use BlobFuse2 instead. // UID defines the user ID (number) which will be set as owner of the mounted volume. - // Deprecated, use BlobFuse2 instead. // +optional UID string `json:"uid,omitempty"` // Optional. Volume mount owner UserID. Used instead of GID. - // TODO: describe + // Deprecated: use BlobFuse2 instead. // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 instead. // +optional SkuName string `json:"skuName,omitempty"` // Available values: Standard_LRS (default), Premium_LRS, Standard_GRS, Standard_RAGRS. https://docs.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - // TODO: describe + // Deprecated: use BlobFuse2 instead. // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 instead. // +optional RequestsStorage string `json:"requestsStorage,omitempty"` // Requests resource storage size. Default "1Mi". https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim + // Deprecated: use BlobFuse2 instead. // Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 instead. // +kubebuilder:validation:Enum=ReadOnlyMany;ReadWriteOnce;ReadWriteMany;"" // +optional AccessMode string `json:"accessMode,omitempty"` // Available values: ReadOnlyMany (default) - read-only by many nodes, ReadWriteOnce - read-write by a single node, ReadWriteMany - read-write by many nodes. https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes + // Deprecated: use BlobFuse2 instead. // Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // Deprecated, use BlobFuse2 instead. // +kubebuilder:validation:Enum=Immediate;WaitForFirstConsumer;"" // +optional BindingMode string `json:"bindingMode,omitempty"` // Volume binding mode. Available values: Immediate (default), WaitForFirstConsumer. https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode @@ -987,10 +985,6 @@ type RadixVolumeMount struct { // BlobFuse2 settings for Azure Storage FUSE CSI driver with the protocol fuse2 BlobFuse2 *RadixBlobFuse2VolumeMount `json:"blobFuse2,omitempty"` - // AzureFile settings for Azure File CSI driver - // Deprecated. - AzureFile *RadixAzureFileVolumeMount `json:"azureFile,omitempty"` - // EmptyDir settings for EmptyDir volume EmptyDir *RadixEmptyDirVolumeMount `json:"emptyDir,omitempty"` } @@ -1083,44 +1077,6 @@ type RadixBlobFuse2VolumeMount struct { ResourceGroup string `json:"resourceGroup,omitempty"` } -// RadixAzureFileVolumeMount defines an external storage resource, configured to use Azure File with CSI driver. -// Deprecated, use BlobFuse2 instead. -type RadixAzureFileVolumeMount struct { - // Share. Name of the file share in the external storage resource. - // +optional - Share string `json:"share,omitempty"` - - // GID defines the group ID (number) which will be set as owner of the mounted volume. - // +optional - GID string `json:"gid,omitempty"` // Optional. Volume mount owner GroupID. Used when drivers do not honor fsGroup securityContext setting. https://github.com/kubernetes-sigs/blob-csi-driver/blob/master/docs/driver-parameters.md - - // UID defines the user ID (number) which will be set as owner of the mounted volume. - // +optional - UID string `json:"uid,omitempty"` // Optional. Volume mount owner UserID. Used instead of GID. - - // SKU Type of Azure storage. - // More info: https://learn.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - // +optional - SkuName string `json:"skuName,omitempty"` // Available values: Standard_LRS (default), Premium_LRS, Standard_GRS, Standard_RAGRS. https://docs.microsoft.com/en-us/rest/api/storagerp/srp_sku_types - - // Requested size (opens new window)of allocated mounted volume. Default value is set to "1Mi" (1 megabyte). Current version of the driver does not affect mounted volume size - // More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - // +optional - RequestsStorage string `json:"requestsStorage,omitempty"` // Requests resource storage size. Default "1Mi". https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim - - // Access mode from a container to an external storage. ReadOnlyMany (default), ReadWriteOnce, ReadWriteMany. - // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // +kubebuilder:validation:Enum=ReadOnlyMany;ReadWriteOnce;ReadWriteMany;"" - // +optional - AccessMode string `json:"accessMode,omitempty"` // Available values: ReadOnlyMany (default) - read-only by many nodes, ReadWriteOnce - read-write by a single node, ReadWriteMany - read-write by many nodes. https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes - - // Binding mode from a container to an external storage. Immediate (default), WaitForFirstConsumer. - // More info: https://www.radix.equinor.com/guides/volume-mounts/optional-settings/ - // +kubebuilder:validation:Enum=Immediate;WaitForFirstConsumer;"" - // +optional - BindingMode string `json:"bindingMode,omitempty"` // Volume binding mode. Available values: Immediate (default), WaitForFirstConsumer. https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode -} - // RadixVolumeMountStreaming configure streaming to read and write large files that will not fit in the file cache on the local disk. Used for blobfuse2. // More info: https://github.com/Azure/azure-storage-fuse/blob/main/STREAMING.md type RadixVolumeMountStreaming struct { diff --git a/pkg/apis/radix/v1/zz_generated.deepcopy.go b/pkg/apis/radix/v1/zz_generated.deepcopy.go index e9d510c85..41e644822 100644 --- a/pkg/apis/radix/v1/zz_generated.deepcopy.go +++ b/pkg/apis/radix/v1/zz_generated.deepcopy.go @@ -980,22 +980,6 @@ func (in *RadixApplyConfigSpec) DeepCopy() *RadixApplyConfigSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RadixAzureFileVolumeMount) DeepCopyInto(out *RadixAzureFileVolumeMount) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RadixAzureFileVolumeMount. -func (in *RadixAzureFileVolumeMount) DeepCopy() *RadixAzureFileVolumeMount { - if in == nil { - return nil - } - out := new(RadixAzureFileVolumeMount) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RadixAzureKeyVault) DeepCopyInto(out *RadixAzureKeyVault) { *out = *in @@ -3221,11 +3205,6 @@ func (in *RadixVolumeMount) DeepCopyInto(out *RadixVolumeMount) { *out = new(RadixBlobFuse2VolumeMount) (*in).DeepCopyInto(*out) } - if in.AzureFile != nil { - in, out := &in.AzureFile, &out.AzureFile - *out = new(RadixAzureFileVolumeMount) - **out = **in - } if in.EmptyDir != nil { in, out := &in.EmptyDir, &out.EmptyDir *out = new(RadixEmptyDirVolumeMount) diff --git a/pkg/apis/radixvalidators/errors.go b/pkg/apis/radixvalidators/errors.go index 0456d0e18..4f366df4f 100644 --- a/pkg/apis/radixvalidators/errors.go +++ b/pkg/apis/radixvalidators/errors.go @@ -9,34 +9,35 @@ import ( ) var ( - ErrMissingPrivateImageHubUsername = errors.New("missing private image hub username") - ErrEnvForDNSAppAliasNotDefined = errors.New("env for dns app alias not defined") - ErrComponentForDNSAppAliasNotDefined = errors.New("component for dns app alias not defined") - ErrExternalAliasCannotBeEmpty = errors.New("external alias cannot be empty") - ErrEnvForDNSExternalAliasNotDefined = errors.New("env for dns external alias not defined") - ErrComponentForDNSExternalAliasNotDefined = errors.New("component for dns external alias not defined") - ErrComponentForDNSExternalAliasIsNotMarkedAsPublic = errors.New("component for dns external alias is not marked as public") - ErrComponentHasInvalidHealthCheck = errors.New("component has invalid health check") - ErrEnvironmentReferencedByComponentDoesNotExist = errors.New("environment referenced by component does not exist") - ErrInvalidPortNameLength = errors.New("invalid port name length") - ErrPortNameIsRequiredForPublicComponent = errors.New("port name is required for public component") - ErrMonitoringPortNameIsNotFoundComponent = errors.New("monitoring port name is not found component") - ErrMultipleMatchingPortNames = errors.New("multiple matching port names") - ErrSchedulerPortCannotBeEmptyForJob = errors.New("scheduler port cannot be empty for job") - ErrPayloadPathCannotBeEmptyForJob = errors.New("payload path cannot be empty for job") - ErrMemoryResourceRequirementFormat = errors.New("memory resource requirement format") - ErrCPUResourceRequirementFormat = errors.New("cpu resource requirement format") - ErrInvalidVerificationType = errors.New("invalid verification") - ErrInvalidHealthCheckProbe = errors.New("probe configuration error, only one action allowed") - ErrSuccessThresholdMustBeOne = errors.New("success threshold must be equal to one") - ErrResourceRequestOverLimit = errors.New("resource request over limit") - ErrInvalidResource = errors.New("invalid resource") - ErrDuplicateExternalAlias = errors.New("duplicate external alias") - ErrInvalidBranchName = errors.New("invalid branch name") - ErrCombiningTriggersWithResourcesIsIllegal = errors.New("combining triggers with resources is invalid") - ErrMaxReplicasForHPANotSetOrZero = errors.New("max replicas for hpa not set or zero") - ErrMinReplicasGreaterThanMaxReplicas = errors.New("min replicas greater than max replicas") - ErrNoScalingResourceSet = errors.New("no scaling resource set") // Deprecated: Replaaced by ErrInvalidTriggerDefinition + ErrMissingPrivateImageHubUsername = errors.New("missing private image hub username") + ErrEnvForDNSAppAliasNotDefined = errors.New("env for dns app alias not defined") + ErrComponentForDNSAppAliasNotDefined = errors.New("component for dns app alias not defined") + ErrExternalAliasCannotBeEmpty = errors.New("external alias cannot be empty") + ErrEnvForDNSExternalAliasNotDefined = errors.New("env for dns external alias not defined") + ErrComponentForDNSExternalAliasNotDefined = errors.New("component for dns external alias not defined") + ErrComponentForDNSExternalAliasIsNotMarkedAsPublic = errors.New("component for dns external alias is not marked as public") + ErrComponentHasInvalidHealthCheck = errors.New("component has invalid health check") + ErrEnvironmentReferencedByComponentDoesNotExist = errors.New("environment referenced by component does not exist") + ErrInvalidPortNameLength = errors.New("invalid port name length") + ErrPortNameIsRequiredForPublicComponent = errors.New("port name is required for public component") + ErrMonitoringPortNameIsNotFoundComponent = errors.New("monitoring port name is not found component") + ErrMultipleMatchingPortNames = errors.New("multiple matching port names") + ErrSchedulerPortCannotBeEmptyForJob = errors.New("scheduler port cannot be empty for job") + ErrPayloadPathCannotBeEmptyForJob = errors.New("payload path cannot be empty for job") + ErrMemoryResourceRequirementFormat = errors.New("memory resource requirement format") + ErrCPUResourceRequirementFormat = errors.New("cpu resource requirement format") + ErrInvalidVerificationType = errors.New("invalid verification") + ErrInvalidHealthCheckProbe = errors.New("probe configuration error, only one action allowed") + ErrSuccessThresholdMustBeOne = errors.New("success threshold must be equal to one") + ErrResourceRequestOverLimit = errors.New("resource request over limit") + ErrInvalidResource = errors.New("invalid resource") + ErrDuplicateExternalAlias = errors.New("duplicate external alias") + ErrInvalidBranchName = errors.New("invalid branch name") + ErrCombiningTriggersWithResourcesIsIllegal = errors.New("combining triggers with resources is invalid") + ErrMaxReplicasForHPANotSetOrZero = errors.New("max replicas for hpa not set or zero") + ErrMinReplicasGreaterThanMaxReplicas = errors.New("min replicas greater than max replicas") + // Deprecated: Replaced by ErrInvalidTriggerDefinition + ErrNoScalingResourceSet = errors.New("no scaling resource set") ErrNoDefinitionInTrigger = errors.New("no definition in trigger") ErrMoreThanOneDefinitionInTrigger = errors.New("each trigger must contain only one definition") ErrInvalidTriggerDefinition = errors.New("invalid trigger definition") diff --git a/pkg/apis/utils/applicationcomponent_builder.go b/pkg/apis/utils/applicationcomponent_builder.go index dae81bd3b..bda82dd4e 100644 --- a/pkg/apis/utils/applicationcomponent_builder.go +++ b/pkg/apis/utils/applicationcomponent_builder.go @@ -13,7 +13,8 @@ type RadixApplicationComponentBuilder interface { WithHealthChecks(startupProbe, readynessProbe, livenessProbe *radixv1.RadixProbe) RadixApplicationComponentBuilder WithImage(string) RadixApplicationComponentBuilder WithImageTagName(imageTagName string) RadixApplicationComponentBuilder - WithPublic(bool) RadixApplicationComponentBuilder // Deprecated: For backwards compatibility WithPublic is still supported, new code should use WithPublicPort instead + // Deprecated: For backwards compatibility WithPublic is still supported, new code should use WithPublicPort instead + WithPublic(bool) RadixApplicationComponentBuilder WithPublicPort(string) RadixApplicationComponentBuilder WithPort(string, int32) RadixApplicationComponentBuilder WithPorts([]radixv1.ComponentPort) RadixApplicationComponentBuilder @@ -44,28 +45,29 @@ type radixApplicationComponentBuilder struct { dockerfileName string image string alwaysPullImageOnDeploy *bool - public bool // Deprecated: For backwards compatibility public is still supported, new code should use publicPort instead - publicPort string - monitoringConfig radixv1.MonitoringConfig - ports []radixv1.ComponentPort - secrets []string - secretRefs radixv1.RadixSecretRefs - ingressConfiguration []string - environmentConfig []RadixEnvironmentConfigBuilder - variables radixv1.EnvVarsMap - resources radixv1.ResourceRequirements - node radixv1.RadixNode - authentication *radixv1.Authentication - volumeMounts []radixv1.RadixVolumeMount - enabled *bool - identity *radixv1.Identity - readOnlyFileSystem *bool - monitoring *bool - imageTagName string - horizontalScaling *radixv1.RadixHorizontalScaling - runtime *radixv1.Runtime - network *radixv1.Network - healtChecks *radixv1.RadixHealthChecks + // Deprecated: For backwards compatibility public is still supported, new code should use publicPort instead + public bool + publicPort string + monitoringConfig radixv1.MonitoringConfig + ports []radixv1.ComponentPort + secrets []string + secretRefs radixv1.RadixSecretRefs + ingressConfiguration []string + environmentConfig []RadixEnvironmentConfigBuilder + variables radixv1.EnvVarsMap + resources radixv1.ResourceRequirements + node radixv1.RadixNode + authentication *radixv1.Authentication + volumeMounts []radixv1.RadixVolumeMount + enabled *bool + identity *radixv1.Identity + readOnlyFileSystem *bool + monitoring *bool + imageTagName string + horizontalScaling *radixv1.RadixHorizontalScaling + runtime *radixv1.Runtime + network *radixv1.Network + healtChecks *radixv1.RadixHealthChecks } func (rcb *radixApplicationComponentBuilder) WithName(name string) RadixApplicationComponentBuilder { From aa5cb3b4a89ca21c9a77aad6ac472551e7681826 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Wed, 22 Jan 2025 13:41:07 +0100 Subject: [PATCH 68/68] Suppressed linter alerts --- pkg/apis/deployment/deployment_test.go | 7 ++----- pkg/apis/deployment/radixcomponent.go | 1 + pkg/apis/deployment/radixcomponent_test.go | 2 +- pkg/apis/radixvalidators/validate_ra.go | 7 +++++++ pkg/apis/radixvalidators/validate_ra_test.go | 1 + pkg/apis/volumemount/volumemount.go | 13 +++++++++++++ pkg/apis/volumemount/volumemount_test.go | 3 ++- 7 files changed, 27 insertions(+), 7 deletions(-) diff --git a/pkg/apis/deployment/deployment_test.go b/pkg/apis/deployment/deployment_test.go index 994028241..b2e1d28cd 100644 --- a/pkg/apis/deployment/deployment_test.go +++ b/pkg/apis/deployment/deployment_test.go @@ -1,4 +1,6 @@ // file deepcode ignore HardcodedPassword/test: unit tests +// +//nolint:staticcheck package deployment import ( @@ -2317,7 +2319,6 @@ func TestObjectSynced_PublicToNonPublic_HandlesChange(t *testing.T) { assert.Equal(t, 0, len(ingresses.Items), "No component should be public") } -//nolint:staticcheck func TestObjectSynced_PublicPort_OldPublic(t *testing.T) { tu, client, kubeUtil, radixclient, kedaClient, prometheusclient, _, certClient := SetupTest(t) defer TeardownTest() @@ -2330,7 +2331,6 @@ func TestObjectSynced_PublicPort_OldPublic(t *testing.T) { WithAppName(anyAppName). WithEnvironment(anyEnvironmentName). WithComponents( - //lint:ignore SA1019 backward compatilibity test utils.NewDeployComponentBuilder(). WithName(componentOneName). WithPort("https", 443). @@ -2350,7 +2350,6 @@ func TestObjectSynced_PublicPort_OldPublic(t *testing.T) { WithAppName(anyAppName). WithEnvironment(anyEnvironmentName). WithComponents( - //lint:ignore SA1019 backward compatilibity test utils.NewDeployComponentBuilder(). WithName(componentOneName). WithPort("https", 443). @@ -2374,7 +2373,6 @@ func TestObjectSynced_PublicPort_OldPublic(t *testing.T) { WithAppName(anyAppName). WithEnvironment(anyEnvironmentName). WithComponents( - //lint:ignore SA1019 backward compatilibity test utils.NewDeployComponentBuilder(). WithName(componentOneName). WithPort("https", 443). @@ -2391,7 +2389,6 @@ func TestObjectSynced_PublicPort_OldPublic(t *testing.T) { WithAppName(anyAppName). WithEnvironment(anyEnvironmentName). WithComponents( - //lint:ignore SA1019 backward compatilibity test utils.NewDeployComponentBuilder(). WithName(componentOneName). WithPort("https", 443). diff --git a/pkg/apis/deployment/radixcomponent.go b/pkg/apis/deployment/radixcomponent.go index 2fee75d33..7fa8a40a7 100644 --- a/pkg/apis/deployment/radixcomponent.go +++ b/pkg/apis/deployment/radixcomponent.go @@ -300,6 +300,7 @@ func getEnvironmentSpecificConfigForComponent(component radixv1.RadixComponent, } func getRadixComponentPort(radixComponent *radixv1.RadixComponent) string { + //nolint:staticcheck if radixComponent.PublicPort == "" && radixComponent.Public { return radixComponent.Ports[0].Name } diff --git a/pkg/apis/deployment/radixcomponent_test.go b/pkg/apis/deployment/radixcomponent_test.go index f787646c7..49293a139 100644 --- a/pkg/apis/deployment/radixcomponent_test.go +++ b/pkg/apis/deployment/radixcomponent_test.go @@ -1,3 +1,4 @@ +//nolint:staticcheck package deployment import ( @@ -175,7 +176,6 @@ func TestGetOAuth2AuthenticationForComponent(t *testing.T) { } } -//nolint:staticcheck func TestGetRadixComponentsForEnv_PublicPort_OldPublic(t *testing.T) { // New publicPort does not exist, old public does not exist componentName := "comp" diff --git a/pkg/apis/radixvalidators/validate_ra.go b/pkg/apis/radixvalidators/validate_ra.go index 31b9b6aca..5062f443d 100644 --- a/pkg/apis/radixvalidators/validate_ra.go +++ b/pkg/apis/radixvalidators/validate_ra.go @@ -137,6 +137,7 @@ func validatePrivateImageHubs(app *radixv1.RadixApplication) error { // RAContainsOldPublic Checks to see if the radix config is using the deprecated config for public port func RAContainsOldPublic(app *radixv1.RadixApplication) bool { for _, component := range app.Spec.Components { + //nolint:staticcheck if component.Public { return true } @@ -580,6 +581,7 @@ func validateVerificationType(verificationType *radixv1.VerificationType) error func componentHasPublicPort(component *radixv1.RadixComponent) bool { return slice.Any(component.GetPorts(), func(p radixv1.ComponentPort) bool { + //nolint:staticcheck return len(p.Name) > 0 && (p.Name == component.PublicPort || component.Public) }) } @@ -1614,14 +1616,18 @@ func validateVolumeMounts(volumeMounts []radixv1.RadixVolumeMount) error { } func validateVolumeMountDeprecatedSource(v *radixv1.RadixVolumeMount) error { + //nolint:staticcheck if v.Type != radixv1.MountTypeBlobFuse2FuseCsiAzure { return volumeMountDeprecatedSourceValidationError(ErrVolumeMountInvalidType) } + //nolint:staticcheck if len(v.RequestsStorage) > 0 { + //nolint:staticcheck if _, err := resource.ParseQuantity(v.RequestsStorage); err != nil { return volumeMountDeprecatedSourceValidationError(fmt.Errorf("%w. %w", ErrVolumeMountInvalidRequestsStorage, err)) } } + //nolint:staticcheck if v.Type == radixv1.MountTypeBlobFuse2FuseCsiAzure && len(v.Container) == 0 { return volumeMountBlobFuse2ValidationError(ErrVolumeMountMissingContainer) } @@ -1717,6 +1723,7 @@ func getEnv(app *radixv1.RadixApplication, name string) *radixv1.Environment { func doesComponentHaveAPublicPort(app *radixv1.RadixApplication, name string) bool { for _, component := range app.Spec.Components { if component.Name == name { + //nolint:staticcheck return component.Public || component.PublicPort != "" } } diff --git a/pkg/apis/radixvalidators/validate_ra_test.go b/pkg/apis/radixvalidators/validate_ra_test.go index d994da3a0..9264f87be 100644 --- a/pkg/apis/radixvalidators/validate_ra_test.go +++ b/pkg/apis/radixvalidators/validate_ra_test.go @@ -1,3 +1,4 @@ +//nolint:staticcheck package radixvalidators_test import ( diff --git a/pkg/apis/volumemount/volumemount.go b/pkg/apis/volumemount/volumemount.go index fdf61974e..572bfad80 100644 --- a/pkg/apis/volumemount/volumemount.go +++ b/pkg/apis/volumemount/volumemount.go @@ -144,6 +144,7 @@ func CreateOrUpdateVolumeMountSecrets(ctx context.Context, kubeUtil *kube.Kube, // GetCsiAzureVolumeMountType Gets the CSI Azure volume mount type func GetCsiAzureVolumeMountType(radixVolumeMount *radixv1.RadixVolumeMount) radixv1.MountType { if radixVolumeMount.BlobFuse2 == nil { + //nolint:staticcheck return radixVolumeMount.Type } switch radixVolumeMount.BlobFuse2.Protocol { @@ -246,9 +247,11 @@ func getCsiRadixVolumeTypeId(radixVolumeMount *radixv1.RadixVolumeMount) (string return "", fmt.Errorf("unknown blobfuse2 protocol %s", radixVolumeMount.BlobFuse2.Protocol) } } + //nolint:staticcheck if radixVolumeMount.Type == radixv1.MountTypeBlobFuse2FuseCsiAzure { return csiVolumeTypeBlobFuse2ProtocolFuse, nil } + //nolint:staticcheck return "", fmt.Errorf("unknown volume mount type %s", radixVolumeMount.Type) } @@ -377,9 +380,11 @@ func getCsiAzureVolumeSource(componentName string, radixVolumeMount *radixv1.Rad } func getComponentVolumeMountDeprecatedVolumeSource(componentName string, volumeMount *radixv1.RadixVolumeMount) (*corev1.VolumeSource, error) { + //nolint:staticcheck if volumeMount.Type == radixv1.MountTypeBlobFuse2FuseCsiAzure { return getCsiAzureVolumeSource(componentName, volumeMount) } + //nolint:staticcheck return nil, fmt.Errorf("unsupported volume type %s", volumeMount.Type) } @@ -403,10 +408,12 @@ func GetVolumeMountVolumeName(volumeMount *radixv1.RadixVolumeMount, componentNa } func getVolumeMountDeprecatedVolumeName(volumeMount *radixv1.RadixVolumeMount, componentName string) (string, error) { + //nolint:staticcheck switch volumeMount.Type { case radixv1.MountTypeBlobFuse2FuseCsiAzure: return getCsiAzureVolumeMountName(componentName, volumeMount) } + //nolint:staticcheck return "", fmt.Errorf("unsupported volume type %s", volumeMount.Type) } @@ -660,6 +667,7 @@ func getStreamingMountOptions(streaming *radixv1.RadixVolumeMountStreaming) []st } func getVolumeMountAccessMode(radixVolumeMount *radixv1.RadixVolumeMount) corev1.PersistentVolumeAccessMode { + //nolint:staticcheck accessMode := radixVolumeMount.AccessMode if radixVolumeMount.BlobFuse2 != nil { accessMode = radixVolumeMount.BlobFuse2.AccessMode @@ -679,6 +687,7 @@ func getRadixBlobFuse2VolumeMountUid(radixVolumeMount *radixv1.RadixVolumeMount) if radixVolumeMount.BlobFuse2 != nil { return radixVolumeMount.BlobFuse2.UID } + //nolint:staticcheck return radixVolumeMount.UID } @@ -686,6 +695,7 @@ func getRadixBlobFuse2VolumeMountGid(radixVolumeMount *radixv1.RadixVolumeMount) if radixVolumeMount.BlobFuse2 != nil { return radixVolumeMount.BlobFuse2.GID } + //nolint:staticcheck return radixVolumeMount.GID } @@ -693,6 +703,7 @@ func getRadixBlobFuse2VolumeMountContainerName(radixVolumeMount *radixv1.RadixVo if radixVolumeMount.BlobFuse2 != nil { return radixVolumeMount.BlobFuse2.Container } + //nolint:staticcheck return radixVolumeMount.Storage } @@ -700,6 +711,7 @@ func getRadixBlobFuse2VolumeMountRequestsStorage(radixVolumeMount *radixv1.Radix if radixVolumeMount.BlobFuse2 != nil { return radixVolumeMount.BlobFuse2.RequestsStorage } + //nolint:staticcheck return radixVolumeMount.RequestsStorage } @@ -730,6 +742,7 @@ func getRadixVolumeMountStorage(radixVolumeMount *radixv1.RadixVolumeMount) stri if len(blobFuse2VolumeMountContainer) != 0 { return blobFuse2VolumeMountContainer } + //nolint:staticcheck return radixVolumeMount.Storage } diff --git a/pkg/apis/volumemount/volumemount_test.go b/pkg/apis/volumemount/volumemount_test.go index 78ff00692..94d58930c 100644 --- a/pkg/apis/volumemount/volumemount_test.go +++ b/pkg/apis/volumemount/volumemount_test.go @@ -1,3 +1,4 @@ +//nolint:staticcheck package volumemount import ( @@ -983,7 +984,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureResources() { }()...) suite.T().Run("CSI Azure volume PVCs and PersistentVolume", func(t *testing.T) { - for _, factory := range suite.radixCommonDeployComponentFactories[:1] { + for _, factory := range suite.radixCommonDeployComponentFactories { for _, scenario := range scenarios { t.Logf("Test case %s, volume type %s, component %s", scenario.name, scenario.props.radixVolumeMountType, factory.GetTargetType()) testEnv := getTestEnv()