From 8d332b770f49bc4639a88b42369b755d3810e209 Mon Sep 17 00:00:00 2001 From: Abhijeet Dargude Date: Thu, 3 Oct 2024 17:27:28 +0000 Subject: [PATCH] Use requiresRepublish in staging manifest. --- ...secretproviderclasspodstatus_controller.go | 104 +++++++----------- ...tproviderclasspodstatus_controller_test.go | 33 +----- deploy/csidriver.yaml | 1 - .../templates/csidriver.yaml | 1 + manifest_staging/deploy/csidriver.yaml | 1 + pkg/secrets-store/nodeserver.go | 24 ++-- pkg/secrets-store/nodeserver_test.go | 19 ++-- pkg/secrets-store/secrets-store.go | 10 +- pkg/secrets-store/utils.go | 9 ++ test/bats/aws.bats | 4 +- test/bats/azure.bats | 2 +- test/bats/e2e-provider.bats | 23 ++-- test/bats/helpers.bash | 2 + test/bats/vault.bats | 11 +- 14 files changed, 109 insertions(+), 135 deletions(-) diff --git a/controllers/secretproviderclasspodstatus_controller.go b/controllers/secretproviderclasspodstatus_controller.go index 8fbee8c14..f4e5dc703 100644 --- a/controllers/secretproviderclasspodstatus_controller.go +++ b/controllers/secretproviderclasspodstatus_controller.go @@ -295,58 +295,44 @@ func (r *SecretProviderClassPodStatusReconciler) Reconcile(ctx context.Context, errs = append(errs, fmt.Errorf("failed to validate secret object in spc %s/%s, err: %w", spc.Namespace, spc.Name, err)) continue } - exists, err := r.secretExists(ctx, secretName, req.Namespace) - if err != nil { - klog.ErrorS(err, "failed to check if secret exists", "secret", klog.ObjectRef{Namespace: req.Namespace, Name: secretName}, "spc", klog.KObj(spc), "pod", klog.KObj(pod), "spcps", klog.KObj(spcPodStatus)) - // syncSecret.enabled is set to false by default in the helm chart for installing the driver in v0.0.23+ - // that would result in a forbidden error, so generate a warning that can be helpful for debugging - if apierrors.IsForbidden(err) { - klog.Warning(SyncSecretForbiddenWarning) - } - errs = append(errs, fmt.Errorf("failed to check if secret %s exists, err: %w", secretName, err)) - continue - } var funcs []func() (bool, error) + secretType := secretutil.GetSecretType(strings.TrimSpace(secretObj.Type)) - if !exists { - secretType := secretutil.GetSecretType(strings.TrimSpace(secretObj.Type)) - - var datamap map[string][]byte - if datamap, err = secretutil.GetSecretData(secretObj.Data, secretType, files); err != nil { - r.generateEvent(pod, corev1.EventTypeWarning, secretCreationFailedReason, fmt.Sprintf("failed to get data in spc %s/%s for secret %s, err: %+v", req.Namespace, spcName, secretName, err)) - klog.ErrorS(err, "failed to get data in spc for secret", "spc", klog.KObj(spc), "pod", klog.KObj(pod), "secret", klog.ObjectRef{Namespace: req.Namespace, Name: secretName}, "spcps", klog.KObj(spcPodStatus)) - errs = append(errs, fmt.Errorf("failed to get data in spc %s/%s for secret %s, err: %w", req.Namespace, spcName, secretName, err)) - continue - } + var datamap map[string][]byte + if datamap, err = secretutil.GetSecretData(secretObj.Data, secretType, files); err != nil { + r.generateEvent(pod, corev1.EventTypeWarning, secretCreationFailedReason, fmt.Sprintf("failed to get data in spc %s/%s for secret %s, err: %+v", req.Namespace, spcName, secretName, err)) + klog.ErrorS(err, "failed to get data in spc for secret", "spc", klog.KObj(spc), "pod", klog.KObj(pod), "secret", klog.ObjectRef{Namespace: req.Namespace, Name: secretName}, "spcps", klog.KObj(spcPodStatus)) + errs = append(errs, fmt.Errorf("failed to get data in spc %s/%s for secret %s, err: %w", req.Namespace, spcName, secretName, err)) + continue + } - labelsMap := make(map[string]string) - if secretObj.Labels != nil { - labelsMap = secretObj.Labels - } - annotationsMap := make(map[string]string) - if secretObj.Annotations != nil { - annotationsMap = secretObj.Annotations - } - // Set secrets-store.csi.k8s.io/managed=true label on the secret that's created and managed - // by the secrets-store-csi-driver. This label will be used to perform a filtered list watch - // only on secrets created and managed by the driver - labelsMap[SecretManagedLabel] = "true" - - createFn := func() (bool, error) { - if err := r.createK8sSecret(ctx, secretName, req.Namespace, datamap, labelsMap, annotationsMap, secretType); err != nil { - klog.ErrorS(err, "failed to create Kubernetes secret", "spc", klog.KObj(spc), "pod", klog.KObj(pod), "secret", klog.ObjectRef{Namespace: req.Namespace, Name: secretName}, "spcps", klog.KObj(spcPodStatus)) - // syncSecret.enabled is set to false by default in the helm chart for installing the driver in v0.0.23+ - // that would result in a forbidden error, so generate a warning that can be helpful for debugging - if apierrors.IsForbidden(err) { - klog.Warning(SyncSecretForbiddenWarning) - } - return false, nil + labelsMap := make(map[string]string) + if secretObj.Labels != nil { + labelsMap = secretObj.Labels + } + annotationsMap := make(map[string]string) + if secretObj.Annotations != nil { + annotationsMap = secretObj.Annotations + } + // Set secrets-store.csi.k8s.io/managed=true label on the secret that's created and managed + // by the secrets-store-csi-driver. This label will be used to perform a filtered list watch + // only on secrets created and managed by the driver + labelsMap[SecretManagedLabel] = "true" + + createFn := func() (bool, error) { + if err := r.createOrUpdateK8sSecret(ctx, secretName, req.Namespace, datamap, labelsMap, annotationsMap, secretType); err != nil { + klog.ErrorS(err, "failed to create Kubernetes secret", "spc", klog.KObj(spc), "pod", klog.KObj(pod), "secret", klog.ObjectRef{Namespace: req.Namespace, Name: secretName}, "spcps", klog.KObj(spcPodStatus)) + // syncSecret.enabled is set to false by default in the helm chart for installing the driver in v0.0.23+ + // that would result in a forbidden error, so generate a warning that can be helpful for debugging + if apierrors.IsForbidden(err) { + klog.Warning(SyncSecretForbiddenWarning) } - return true, nil + return false, nil } - funcs = append(funcs, createFn) + return true, nil } + funcs = append(funcs, createFn) for _, f := range funcs { if err := wait.ExponentialBackoff(wait.Backoff{ @@ -410,9 +396,9 @@ func (r *SecretProviderClassPodStatusReconciler) processIfBelongsToNode(objMeta return true } -// createK8sSecret creates K8s secret with data from mounted files +// createOrUpdateK8sSecret creates K8s secret with data from mounted files // If a secret with the same name already exists in the namespace of the pod, the error is nil. -func (r *SecretProviderClassPodStatusReconciler) createK8sSecret(ctx context.Context, name, namespace string, datamap map[string][]byte, labelsmap map[string]string, annotationsmap map[string]string, secretType corev1.SecretType) error { +func (r *SecretProviderClassPodStatusReconciler) createOrUpdateK8sSecret(ctx context.Context, name, namespace string, datamap map[string][]byte, labelsmap map[string]string, annotationsmap map[string]string, secretType corev1.SecretType) error { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, @@ -430,6 +416,13 @@ func (r *SecretProviderClassPodStatusReconciler) createK8sSecret(ctx context.Con return nil } if apierrors.IsAlreadyExists(err) { + klog.InfoS("Kubernetes secret is already created", "secret", klog.ObjectRef{Namespace: namespace, Name: name}) + err := r.writer.Update(ctx, secret) + if err != nil { + klog.Errorf("Unable to update kubernetes secret", "secret", klog.ObjectRef{Namespace: namespace, Name: name}) + return err + } + klog.InfoS("successfully updated Kubernetes secret", "secret", klog.ObjectRef{Namespace: namespace, Name: name}) return nil } return err @@ -477,23 +470,6 @@ func (r *SecretProviderClassPodStatusReconciler) patchSecretWithOwnerRef(ctx con return nil } -// secretExists checks if the secret with name and namespace already exists -func (r *SecretProviderClassPodStatusReconciler) secretExists(ctx context.Context, name, namespace string) (bool, error) { - o := &corev1.Secret{} - secretKey := types.NamespacedName{ - Namespace: namespace, - Name: name, - } - err := r.Client.Get(ctx, secretKey, o) - if err == nil { - return true, nil - } - if apierrors.IsNotFound(err) { - return false, nil - } - return false, err -} - // generateEvent generates an event func (r *SecretProviderClassPodStatusReconciler) generateEvent(obj apiruntime.Object, eventType, reason, message string) { if obj != nil { diff --git a/controllers/secretproviderclasspodstatus_controller_test.go b/controllers/secretproviderclasspodstatus_controller_test.go index 11c4bab30..0558acee6 100644 --- a/controllers/secretproviderclasspodstatus_controller_test.go +++ b/controllers/secretproviderclasspodstatus_controller_test.go @@ -121,31 +121,6 @@ func newReconciler(client client.Client, scheme *runtime.Scheme, nodeID string) } } -func TestSecretExists(t *testing.T) { - g := NewWithT(t) - - scheme, err := setupScheme() - g.Expect(err).NotTo(HaveOccurred()) - - labels := map[string]string{"environment": "test"} - annotations := map[string]string{"kubed.appscode.com/sync": "app=test"} - - initObjects := []client.Object{ - newSecret("my-secret", "default", labels, annotations), - } - - client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjects...).Build() - reconciler := newReconciler(client, scheme, "node1") - - exists, err := reconciler.secretExists(context.TODO(), "my-secret", "default") - g.Expect(exists).To(Equal(true)) - g.Expect(err).NotTo(HaveOccurred()) - - exists, err = reconciler.secretExists(context.TODO(), "my-secret2", "default") - g.Expect(exists).To(Equal(false)) - g.Expect(err).NotTo(HaveOccurred()) -} - func TestPatchSecretWithOwnerRef(t *testing.T) { g := NewWithT(t) @@ -183,7 +158,7 @@ func TestPatchSecretWithOwnerRef(t *testing.T) { g.Expect(secret.GetOwnerReferences()).To(HaveLen(1)) } -func TestCreateK8sSecret(t *testing.T) { +func TestCreateOrUpdateK8sSecret(t *testing.T) { g := NewWithT(t) scheme, err := setupScheme() @@ -198,11 +173,11 @@ func TestCreateK8sSecret(t *testing.T) { client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjects...).Build() reconciler := newReconciler(client, scheme, "node1") - // secret already exists - err = reconciler.createK8sSecret(context.TODO(), "my-secret", "default", nil, labels, annotations, corev1.SecretTypeOpaque) + // secret already exists, just update it. + err = reconciler.createOrUpdateK8sSecret(context.TODO(), "my-secret", "default", nil, labels, annotations, corev1.SecretTypeOpaque) g.Expect(err).NotTo(HaveOccurred()) - err = reconciler.createK8sSecret(context.TODO(), "my-secret2", "default", nil, labels, annotations, corev1.SecretTypeOpaque) + err = reconciler.createOrUpdateK8sSecret(context.TODO(), "my-secret2", "default", nil, labels, annotations, corev1.SecretTypeOpaque) g.Expect(err).NotTo(HaveOccurred()) secret := &corev1.Secret{} err = client.Get(context.TODO(), types.NamespacedName{Name: "my-secret2", Namespace: "default"}, secret) diff --git a/deploy/csidriver.yaml b/deploy/csidriver.yaml index 19b95c1cc..ac3445923 100644 --- a/deploy/csidriver.yaml +++ b/deploy/csidriver.yaml @@ -7,4 +7,3 @@ spec: attachRequired: false volumeLifecycleModes: - Ephemeral - requiresRepublish: true diff --git a/manifest_staging/charts/secrets-store-csi-driver/templates/csidriver.yaml b/manifest_staging/charts/secrets-store-csi-driver/templates/csidriver.yaml index 5aef2b946..830b5be96 100644 --- a/manifest_staging/charts/secrets-store-csi-driver/templates/csidriver.yaml +++ b/manifest_staging/charts/secrets-store-csi-driver/templates/csidriver.yaml @@ -11,6 +11,7 @@ spec: volumeLifecycleModes: - Ephemeral {{- if and (semverCompare ">=1.20-0" .Capabilities.KubeVersion.Version) .Values.tokenRequests }} + requiresRepublish: true tokenRequests: {{- toYaml .Values.tokenRequests | nindent 2 }} {{- end }} diff --git a/manifest_staging/deploy/csidriver.yaml b/manifest_staging/deploy/csidriver.yaml index ac3445923..19b95c1cc 100644 --- a/manifest_staging/deploy/csidriver.yaml +++ b/manifest_staging/deploy/csidriver.yaml @@ -7,3 +7,4 @@ spec: attachRequired: false volumeLifecycleModes: - Ephemeral + requiresRepublish: true diff --git a/pkg/secrets-store/nodeserver.go b/pkg/secrets-store/nodeserver.go index 4a9cc9f41..c4f6ba488 100644 --- a/pkg/secrets-store/nodeserver.go +++ b/pkg/secrets-store/nodeserver.go @@ -93,15 +93,6 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis ns.reporter.ReportNodePublishCtMetric(ctx, providerName) }() - if ns.rotationConfig.enabled { - // Node server retries nodepublish volume continuously, so to rotate secret after every `nextRotationTime`, - // nodeserver should skip secret mount till the next `nextRotationTime` - if ns.rotationConfig.nextRotationTime.After(startTime) { - return &csi.NodePublishVolumeResponse{}, nil - } - ns.rotationConfig.nextRotationTime = ns.rotationConfig.nextRotationTime.Add(ns.rotationConfig.interval) - } - // Check arguments if req.GetVolumeCapability() == nil { return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request") @@ -128,6 +119,20 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis podNamespace = attrib[CSIPodNamespace] podUID = attrib[CSIPodUID] + klog.InfoS("Checking object", "targetPath", targetPath, "pod", klog.ObjectRef{Namespace: podNamespace, Name: podName}) + + if ns.rotationConfig.enabled { + lastModificationTime, err := ns.getLastUpdateTime(targetPath) + if err != nil { + klog.Infof("could not find last modification time for %s, error: %v\n", targetPath, err) + } else if startTime.Before(lastModificationTime.Add(ns.rotationConfig.interval)) { + // if next rotation is not yet, then skip the mount operation + return &csi.NodePublishVolumeResponse{}, nil + } + } + + klog.InfoS("Processing object", "targetPath", targetPath, "pod", klog.ObjectRef{Namespace: podNamespace, Name: podName}) + mounted, err = ns.ensureMountPoint(targetPath) if err != nil { // kubelet will not create the CSI NodePublishVolume target directory in 1.20+, in accordance with the CSI specification. @@ -198,7 +203,6 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if parameters[CSIPodServiceAccountTokens] == "" { // Inject pod service account token into volume attributes klog.Error("csi.storage.k8s.io/serviceAccount.tokens is not populated, set RequiresRepublish") - } // ensure it's read-only diff --git a/pkg/secrets-store/nodeserver_test.go b/pkg/secrets-store/nodeserver_test.go index 0ad7738be..8a6f1f6bc 100644 --- a/pkg/secrets-store/nodeserver_test.go +++ b/pkg/secrets-store/nodeserver_test.go @@ -297,9 +297,8 @@ func TestNodePublishVolume(t *testing.T) { }, }, rotationConfig: &RotationConfig{ - enabled: false, - nextRotationTime: time.Now(), - interval: time.Minute, + enabled: false, + interval: time.Minute, }, }, { @@ -332,9 +331,8 @@ func TestNodePublishVolume(t *testing.T) { }, }, rotationConfig: &RotationConfig{ - enabled: true, - nextRotationTime: time.Now().Add(-3 * time.Minute), // so that rotation period is passed and secret will be mounted. - interval: time.Minute, + enabled: true, + interval: time.Minute, }, }, { @@ -364,9 +362,8 @@ func TestNodePublishVolume(t *testing.T) { }, }, rotationConfig: &RotationConfig{ - enabled: true, - nextRotationTime: time.Now().Add(2 * time.Minute), - interval: time.Minute, + enabled: true, + interval: time.Minute, }, }, } @@ -410,8 +407,8 @@ func TestNodePublishVolume(t *testing.T) { t.Fatalf("expected err to be nil, got: %v", err) } expectedMounts := 1 - if ns.rotationConfig.enabled && ns.rotationConfig.nextRotationTime.After(time.Now()) { - // If rotation time is not reached, there should not be any mounts. + if ns.rotationConfig.enabled { + // If rotation time is not enabled, there should not be any mounts. expectedMounts = 0 } if len(mnts) != expectedMounts { diff --git a/pkg/secrets-store/secrets-store.go b/pkg/secrets-store/secrets-store.go index 7d32542bc..7bc0fc096 100644 --- a/pkg/secrets-store/secrets-store.go +++ b/pkg/secrets-store/secrets-store.go @@ -40,9 +40,8 @@ type SecretsStore struct { // RotationConfig stores the informarmation required to rotate the secrets. type RotationConfig struct { - enabled bool - interval time.Duration - nextRotationTime time.Time + enabled bool + interval time.Duration } func NewSecretsStoreDriver(driverName, nodeID, endpoint string, @@ -92,9 +91,8 @@ func newNodeServer(nodeID string, func NewRotationConfig(enabled bool, interval time.Duration) *RotationConfig { return &RotationConfig{ - enabled: enabled, - interval: interval, - nextRotationTime: time.Now(), + enabled: enabled, + interval: interval, } } diff --git a/pkg/secrets-store/utils.go b/pkg/secrets-store/utils.go index e709b8033..af3799785 100644 --- a/pkg/secrets-store/utils.go +++ b/pkg/secrets-store/utils.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "strings" + "time" secretsstorev1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" "sigs.k8s.io/secrets-store-csi-driver/pkg/util/runtimeutil" @@ -75,6 +76,14 @@ func (ns *nodeServer) ensureMountPoint(target string) (bool, error) { return false, nil } +func (ns *nodeServer) getLastUpdateTime(target string) (time.Time, error) { + info, err := os.Stat(target) + if err != nil { + return time.Time{}, err + } + return info.ModTime(), nil +} + // getSecretProviderItem returns the secretproviderclass object by name and namespace func getSecretProviderItem(ctx context.Context, c client.Client, name, namespace string) (*secretsstorev1.SecretProviderClass, error) { spc := &secretsstorev1.SecretProviderClass{} diff --git a/test/bats/aws.bats b/test/bats/aws.bats index 4144ccb0c..ebb82f0d2 100644 --- a/test/bats/aws.bats +++ b/test/bats/aws.bats @@ -81,7 +81,7 @@ teardown_file() { [[ "${result//$'\r'}" == "BeforeRotation" ]] aws ssm put-parameter --name $PM_ROTATION_TEST_NAME --value AfterRotation --type SecureString --overwrite --region $REGION - sleep 40 + sleep 180 result=$(kubectl --namespace $NAMESPACE exec $POD_NAME -- cat /mnt/secrets-store/$PM_ROTATION_TEST_NAME) [[ "${result//$'\r'}" == "AfterRotation" ]] } @@ -91,7 +91,7 @@ teardown_file() { [[ "${result//$'\r'}" == "BeforeRotation" ]] aws secretsmanager put-secret-value --secret-id $SM_ROT_TEST_NAME --secret-string AfterRotation --region $REGION - sleep 40 + sleep 180 result=$(kubectl --namespace $NAMESPACE exec $POD_NAME -- cat /mnt/secrets-store/$SM_ROT_TEST_NAME) [[ "${result//$'\r'}" == "AfterRotation" ]] } diff --git a/test/bats/azure.bats b/test/bats/azure.bats index 42b37e9f1..09778b2ba 100644 --- a/test/bats/azure.bats +++ b/test/bats/azure.bats @@ -193,7 +193,7 @@ setup() { assert_success envsubst < $BATS_TESTS_DIR/deployment-synck8s-azure.yaml | kubectl apply -n negative-test-ns -f - - sleep 5 + sleep 30 POD=$(kubectl get pod -l app=busybox -n negative-test-ns -o jsonpath="{.items[0].metadata.name}") cmd="kubectl describe pod $POD -n negative-test-ns | grep 'FailedMount.*failed to get secretproviderclass negative-test-ns/azure-sync.*not found'" diff --git a/test/bats/e2e-provider.bats b/test/bats/e2e-provider.bats index a73b27200..e074f7c1b 100644 --- a/test/bats/e2e-provider.bats +++ b/test/bats/e2e-provider.bats @@ -160,10 +160,15 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) } @test "CSI inline volume test with pod portability - read secret from pod" { + # wait for secrets to mount + sleep 180 wait_for_process $WAIT_TIME $SLEEP_TIME "kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$SECRET_NAME | grep '${SECRET_VALUE}'" result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$SECRET_NAME) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] + + sleep 10 + archive_info } @test "CSI inline volume test with pod portability - read key from pod" { @@ -269,7 +274,7 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) envsubst < $BATS_TESTS_DIR/deployment-synck8s-e2e-provider.yaml | kubectl apply -n test-ns -f - - kubectl wait --for=condition=Ready --timeout=60s pod -l app=busybox -n test-ns + kubectl wait --for=condition=Ready --timeout=180s pod -l app=busybox -n test-ns } @test "Test Namespaced scope SecretProviderClass - Sync with K8s secrets - read secret from pod, read K8s secret, read env var, check secret ownerReferences" { @@ -337,13 +342,14 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) @test "deploy pod with multiple secret provider class" { envsubst < $BATS_TESTS_DIR/pod-e2e-provider-inline-volume-multiple-spc.yaml | kubectl apply -f - - kubectl wait --for=condition=Ready --timeout=60s pod/secrets-store-inline-multiple-crd + kubectl wait --for=condition=Ready --timeout=180s pod/secrets-store-inline-multiple-crd run kubectl get pod/secrets-store-inline-multiple-crd assert_success } @test "CSI inline volume test with multiple secret provider class" { + sleep 180 result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/$SECRET_NAME) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] @@ -383,7 +389,7 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) envsubst < $BATS_TESTS_DIR/rotation/e2e_provider_synck8s_v1_secretproviderclass.yaml | kubectl apply -n rotation -f - envsubst < $BATS_TESTS_DIR/rotation/pod-synck8s-e2e-provider.yaml | kubectl apply -n rotation -f - - kubectl wait -n rotation --for=condition=Ready --timeout=60s pod/secrets-store-inline-rotation + kubectl wait -n rotation --for=condition=Ready --timeout=180s pod/secrets-store-inline-rotation run kubectl get pod/secrets-store-inline-rotation -n rotation assert_success @@ -399,11 +405,15 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) # enable rotation response in mock server local curl_pod_name=curl-$(openssl rand -hex 5) kubectl run ${curl_pod_name} -n rotation --image=curlimages/curl:7.75.0 --labels="test=rotation" -- tail -f /dev/null - kubectl wait -n rotation --for=condition=Ready --timeout=60s pod ${curl_pod_name} + kubectl wait -n rotation --for=condition=Ready --timeout=180s pod ${curl_pod_name} local pod_ip=$(kubectl get pod -n kube-system -l app=csi-secrets-store-e2e-provider -o jsonpath="{.items[0].status.podIP}") run kubectl exec ${curl_pod_name} -n rotation -- curl http://${pod_ip}:8080/rotation?rotated=true - sleep 60 + # wait for rotated secret to be mounted + sleep 180 + + # Save logs in case of failure in rotation + archive_info result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/$SECRET_NAME) [[ "${result//$'\r'}" == "rotated" ]] @@ -419,13 +429,12 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) kubectl create ns metrics local curl_pod_name=curl-$(openssl rand -hex 5) kubectl run ${curl_pod_name} -n metrics --image=curlimages/curl:7.75.0 --labels="test=metrics" -- tail -f /dev/null - kubectl wait -n metrics --for=condition=Ready --timeout=60s pod ${curl_pod_name} + kubectl wait -n metrics --for=condition=Ready --timeout=120s pod ${curl_pod_name} for pod_ip in $(kubectl get pod -n kube-system -l app=secrets-store-csi-driver -o jsonpath="{.items[0].status.podIP}") do run kubectl exec ${curl_pod_name} -n metrics -- curl http://${pod_ip}:8095/metrics assert_match "node_publish_total" "${output}" assert_match "node_unpublish_total" "${output}" - assert_match "rotation_reconcile_total" "${output}" done # keeping metrics ns in upgrade tests has no relevance # the namespace is only to run a curl pod to validate metrics diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index 8e95e0c8f..9dd70b893 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -110,6 +110,8 @@ archive_info() { # print detailed pod information kubectl describe pods --all-namespaces > ${LOGS_DIR}/pods-describe.txt + kubectl describe csidriver secrets-store.csi.k8s.io > ${LOGS_DIR}/csidriver-describe.txt + # print logs from the CSI Driver # # assumes driver is installed with helm into the `kube-system` namespace which diff --git a/test/bats/vault.bats b/test/bats/vault.bats index 6291113f2..d38735956 100644 --- a/test/bats/vault.bats +++ b/test/bats/vault.bats @@ -88,11 +88,12 @@ EOF # deploy pod kubectl apply -f $BATS_TESTS_DIR/pod-vault-rotation.yaml - kubectl wait --for=condition=Ready --timeout=60s pod/secrets-store-rotation + kubectl wait --for=condition=Ready --timeout=180s pod/secrets-store-rotation run kubectl get pod/secrets-store-rotation assert_success + sleep 180 # verify starting value result=$(kubectl exec secrets-store-rotation -- cat /mnt/secrets-store/foo) [[ "$result" == "start" ]] @@ -100,11 +101,13 @@ EOF # update the secret value kubectl exec vault-0 --namespace=vault -- vault kv put secret/rotation foo=rotated - sleep 60 + sleep 180 # verify rotated value result=$(kubectl exec secrets-store-rotation -- cat /mnt/secrets-store/foo) [[ "$result" == "rotated" ]] + + archive_info } @test "CSI inline volume test with pod portability - unmount succeeds" { @@ -209,7 +212,7 @@ EOF kubectl apply -n test-ns -f $BATS_TESTS_DIR/deployment-synck8s.yaml - kubectl wait --for=condition=Ready --timeout=90s pod -l app=busybox -n test-ns + kubectl wait --for=condition=Ready --timeout=180s pod -l app=busybox -n test-ns } @test "Test Namespaced scope SecretProviderClass - Sync with K8s secrets - read secret from pod, read K8s secret, read env var, check secret ownerReferences" { @@ -269,7 +272,7 @@ EOF @test "deploy pod with multiple secret provider class" { kubectl apply -f $BATS_TESTS_DIR/pod-vault-inline-volume-multiple-spc.yaml - kubectl wait --for=condition=Ready --timeout=90s pod/secrets-store-inline-multiple-crd + kubectl wait --for=condition=Ready --timeout=180s pod/secrets-store-inline-multiple-crd run kubectl get pod/secrets-store-inline-multiple-crd assert_success