Merge pull request #1498 from gl-johnson/conjur-e2e

test: add e2e for conjur csi provider

Makefile

@@ -437,7 +437,8 @@ e2e-helm-deploy:
 		--set enableSecretRotation=true \
 		--set rotationPollInterval=30s \
 		--set tokenRequests[0].audience="aud1" \
-		--set tokenRequests[1].audience="aud2"
+		--set tokenRequests[1].audience="aud2" \
+		--set tokenRequests[2].audience="conjur"
 
 .PHONY: e2e-helm-upgrade
 e2e-helm-upgrade:
@@ -494,6 +495,10 @@ e2e-gcp:
 e2e-aws:
 	bats -t test/bats/aws.bats
 
+.PHONY: e2e-conjur
+e2e-conjur:
+	bats -t test/bats/conjur.bats
+
 ## --------------------------------------
 ## Generate
 ## --------------------------------------

test/bats/conjur.bats

@@ -0,0 +1,146 @@
+#!/usr/bin/env bats
+
+load helpers
+
+BATS_TESTS_DIR=test/bats/tests/conjur
+WAIT_TIME=180
+SLEEP_TIME=1
+
+CONJUR_NAMESPACE=conjur
+CONJUR_DATA_KEY="$(openssl rand -base64 32)"
+CONJUR_ACCOUNT=default
+CONJUR_URL=conjur-conjur-oss.conjur.svc.cluster.local
+
+EXPECTED_USERNAME="some_user"
+EXPECTED_PASSWORD="SecretPassword1234!"
+
+@test "install conjur provider" { 
+  # Update Helm repos
+  helm repo add cyberark https://cyberark.github.io/helm-charts || true
+  helm repo update
+
+  # Install Conjur
+  helm install conjur cyberark/conjur-oss --namespace=$CONJUR_NAMESPACE --wait --timeout ${WAIT_TIME}s --create-namespace \
+    --set dataKey=$CONJUR_DATA_KEY \
+    --set authenticators="authn\,authn-jwt/kube" \
+    --set service.external.enabled=false
+
+  # Install Conjur CSI Provider
+  helm install conjur-csi-provider cyberark/conjur-k8s-csi-provider --wait --timeout ${WAIT_TIME}s --namespace=kube-system \
+    --set providerServer.image.tag=0.1.0
+} 
+
+@test "setup conjur" { 
+  # Create Conjur account and store admin API key
+  admin_api_key="$(kubectl exec deployment/conjur-conjur-oss \
+    --namespace=$CONJUR_NAMESPACE \
+    --container=conjur-oss \
+    -- conjurctl account create $CONJUR_ACCOUNT | grep API | awk '{print $5}')"
+
+  # Create a Conjur CLI pod
+  kubectl run conjur-cli-pod --image=cyberark/conjur-cli:8 --namespace=$CONJUR_NAMESPACE --command -- sleep infinity
+  kubectl wait --for=condition=ready --timeout=${WAIT_TIME}s --namespace=$CONJUR_NAMESPACE pod/conjur-cli-pod
+
+  # Get values required by authn-jwt authenticator
+  ISSUER=$(kubectl get --raw /.well-known/openid-configuration | jq -r .issuer)
+  JWKS='{"type":"jwks","value":'$(kubectl get --raw /openid/v1/jwks)'}'
+
+  # Copy policy files into CLI container and configure Conjur
+  kubectl -n "${CONJUR_NAMESPACE}" cp $BATS_TESTS_DIR/policy conjur-cli-pod:/policy -c conjur-cli-pod
+  kubectl -n "${CONJUR_NAMESPACE}" exec conjur-cli-pod -- /bin/sh -c "
+  set -x
+  # Initialise CLI and login
+  echo yes | conjur init -u 'https://$CONJUR_URL' -a '$CONJUR_ACCOUNT' --self-signed
+  conjur login -i admin -p $admin_api_key
+
+  # Apply policy
+  conjur policy replace -b root -f /policy/host.yaml
+  conjur policy load -b root -f /policy/authn-jwt.yaml
+  conjur policy load -b root -f /policy/variables.yaml
+  # Set test secret values
+  conjur variable set -i db-credentials/username -v '$EXPECTED_USERNAME'
+  conjur variable set -i db-credentials/password -v '$EXPECTED_PASSWORD'
+  # Set variable values on authenticator
+  conjur variable set -i conjur/authn-jwt/kube/public-keys -v '$JWKS'
+  conjur variable set -i conjur/authn-jwt/kube/issuer -v '$ISSUER'
+  "
+}
+
+@test "deploy conjur secretproviderclass crd" {
+  CONJUR_POD=$(kubectl get pods --namespace=$CONJUR_NAMESPACE -l "app=conjur-oss" -o=jsonpath='{.items[0].metadata.name}')
+  export CONJUR_SSL_CERT=$(kubectl exec --namespace=$CONJUR_NAMESPACE  -c conjur-oss $CONJUR_POD  -- sh -c "openssl s_client -showcerts -connect $CONJUR_URL:443 </dev/null 2>/dev/null | sed -n '/-----BEGIN CERTIFICATE-----/,/-----END CERTIFICATE-----/p'")
+  envsubst < $BATS_TESTS_DIR/conjur_v1_secretproviderclass.yaml | sed '/^ *-----BEGIN CERTIFICATE-----/,$s/^/      /' | kubectl apply -f -
+
+  kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io
+
+  cmd="kubectl get secretproviderclasses.secrets-store.csi.x-k8s.io/conjur -o yaml | grep conjur"
+  wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
+}
+
+@test "CSI inline volume test with pod portability" {
+  kubectl apply -f $BATS_TESTS_DIR/pod-secrets-store-inline-volume-crd.yaml
+
+  kubectl wait --for=condition=Ready --timeout=60s pod/secrets-store-inline-crd
+
+  run kubectl get pod/secrets-store-inline-crd
+  assert_success
+}
+
+@test "CSI inline volume test with pod portability - read conjur secret from pod" {
+  result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/relative/path/username)
+  [[ "${result}" == "$EXPECTED_USERNAME" ]]
+
+  result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/relative/path/password)
+  [[ "${result}" == "$EXPECTED_PASSWORD" ]]
+}
+
+@test "CSI inline volume test with pod portability - rotation succeeds" {
+  # Set initial value
+  kubectl -n conjur exec conjur-cli-pod -- conjur variable set -i db-credentials/username -v initial_value
+
+  # Deploy pod
+  CONJUR_POD=$(kubectl get pods --namespace=$CONJUR_NAMESPACE -l "app=conjur-oss" -o=jsonpath='{.items[0].metadata.name}')
+  export CONJUR_SSL_CERT=$(kubectl exec --namespace=$CONJUR_NAMESPACE -c conjur-oss $CONJUR_POD  -- sh -c "openssl s_client -showcerts -connect $CONJUR_URL:443 </dev/null 2>/dev/null | sed -n '/-----BEGIN CERTIFICATE-----/,/-----END CERTIFICATE-----/p'")
+  envsubst < $BATS_TESTS_DIR/pod-conjur-rotation.yaml | sed '/^ *-----BEGIN CERTIFICATE-----/,$s/^/      /' | kubectl apply -f -
+  kubectl wait --for=condition=Ready --timeout=60s pod/secrets-store-inline-rotation
+
+  run kubectl get pod/secrets-store-inline-rotation
+  assert_success
+
+  # Verify initial value
+  result=$(kubectl exec secrets-store-inline-rotation -- cat /mnt/secrets-store/relative/path/username)
+  [[ "${result}" == "initial_value" ]]
+
+  # Update the secret value and wait for rotation interval
+  kubectl -n conjur exec conjur-cli-pod -- conjur variable set -i db-credentials/username -v rotated_value
+  sleep 40
+
+  # Verify rotated value
+  result=$(kubectl exec secrets-store-inline-rotation -- cat /mnt/secrets-store/relative/path/username)
+  [[ "${result}" == "rotated_value" ]]
+}
+
+@test "CSI inline volume test with pod portability - unmount succeeds" {
+  # On Linux a failure to unmount the tmpfs will block the pod from being deleted.
+  run kubectl delete pod secrets-store-inline-crd
+  assert_success
+
+  run kubectl wait --for=delete --timeout=${WAIT_TIME}s pod/secrets-store-inline-crd
+
+  # Sleep to allow time for logs to propagate.
+  sleep 10
+
+  # save debug information to archive in case of failure
+  archive_info
+
+  # On Windows, the failed unmount calls from: https://github.com/kubernetes-sigs/secrets-store-csi-driver/pull/545
+  # do not prevent the pod from being deleted. Search through the driver logs
+  # for the error.
+  run bash -c "kubectl logs -l app=secrets-store-csi-driver --tail -1 -c secrets-store -n kube-system | grep '^E.*failed to clean and unmount target path.*$'"
+  assert_failure
+}
+
+teardown_file() {
+  archive_provider "app=csi-secrets-store-provider-conjur" || true
+  archive_info || true
+}

test/bats/tests/conjur/conjur_v1_secretproviderclass.yaml

@@ -0,0 +1,16 @@
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: conjur
+spec:
+  provider: conjur
+  parameters:
+    account: default
+    applianceUrl: https://conjur-conjur-oss.conjur.svc.cluster.local
+    authnId: authn-jwt/kube
+    identity: host/workload-host
+    secrets: |
+      - "relative/path/username": "db-credentials/username"
+      - "relative/path/password": "db-credentials/password"
+    sslCertificate: |
+$CONJUR_SSL_CERT

test/bats/tests/conjur/pod-conjur-rotation.yaml

@@ -0,0 +1,40 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  name: secrets-store-inline-rotation
+spec:
+  containers:
+    - image: registry.k8s.io/e2e-test-images/busybox:1.29-4
+      name: busybox
+      imagePullPolicy: IfNotPresent
+      command:
+        - "/bin/sleep"
+        - "10000"
+      volumeMounts:
+        - name: secrets-store-inline
+          mountPath: "/mnt/secrets-store"
+          readOnly: true
+  volumes:
+    - name: secrets-store-inline
+      csi:
+        driver: secrets-store.csi.k8s.io
+        readOnly: true
+        volumeAttributes:
+          secretProviderClass: "conjur-rotation"
+---
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: conjur-rotation
+spec:
+  provider: conjur
+  parameters:
+    account: default
+    applianceUrl: https://conjur-conjur-oss.conjur.svc.cluster.local
+    authnId: authn-jwt/kube
+    identity: host/workload-host
+    secrets: |
+      - "relative/path/username": "db-credentials/username"
+      - "relative/path/password": "db-credentials/password"
+    sslCertificate: |
+$CONJUR_SSL_CERT

test/bats/tests/conjur/pod-secrets-store-inline-volume-crd.yaml

@@ -0,0 +1,23 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  name: secrets-store-inline-crd
+spec:
+  containers:
+  - image: registry.k8s.io/e2e-test-images/busybox:1.29-4
+    name: busybox
+    imagePullPolicy: IfNotPresent
+    command:
+    - "/bin/sleep"
+    - "10000"
+    volumeMounts:
+    - name: secrets-store-inline
+      mountPath: "/mnt/secrets-store"
+      readOnly: true
+  volumes:
+    - name: secrets-store-inline
+      csi:
+        driver: secrets-store.csi.k8s.io
+        readOnly: true
+        volumeAttributes:
+          secretProviderClass: conjur

test/bats/tests/conjur/policy/authn-jwt.yaml

@@ -0,0 +1,13 @@
+---
+- !policy
+  id: conjur/authn-jwt/kube
+  body:
+  - !webservice
+
+  - !variable public-keys
+  - !variable issuer
+
+  - !permit
+    role: !host /workload-host
+    privilege: [ read, authenticate ]
+    resource: !webservice

test/bats/tests/conjur/policy/host.yaml

@@ -0,0 +1,6 @@
+---
+- !host
+  id: workload-host
+  annotations:
+    authn-jwt/kube/kubernetes.io/namespace: default
+    authn-jwt/kube/kubernetes.io/serviceaccount/name: default

test/bats/tests/conjur/policy/variables.yaml

@@ -0,0 +1,12 @@
+---
+- !policy
+  id: db-credentials
+  body:
+  - &variables
+    - !variable username
+    - !variable password
+
+  - !permit
+    role: !host /workload-host
+    privileges: [ read, execute ]
+    resource: *variables