diff --git a/CHANGELOG.md b/CHANGELOG.md index 14433d809ac..f3c862b3342 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ Here is an overview of all new **experimental** features: ### Improvements +- General: Add SecretKey to AWS SecretsManager TriggerAuthentication to allow parsing JSON / Key/Value Pairs in secrets (#5940) - TODO ([#XXX](https://github.com/kedacore/keda/issues/XXX)) ### Fixes diff --git a/apis/keda/v1alpha1/triggerauthentication_types.go b/apis/keda/v1alpha1/triggerauthentication_types.go index 0b0d9ffa315..7685e122660 100644 --- a/apis/keda/v1alpha1/triggerauthentication_types.go +++ b/apis/keda/v1alpha1/triggerauthentication_types.go @@ -376,6 +376,8 @@ type AwsSecretManagerSecret struct { VersionID string `json:"versionId,omitempty"` // +optional VersionStage string `json:"versionStage,omitempty"` + // +optional + SecretKey string `json:"secretKey,omitempty"` } func init() { diff --git a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml index d8a74647ffc..af6e07c158f 100644 --- a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml +++ b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml @@ -183,6 +183,8 @@ spec: type: string parameter: type: string + secretKey: + type: string versionId: type: string versionStage: diff --git a/config/crd/bases/keda.sh_triggerauthentications.yaml b/config/crd/bases/keda.sh_triggerauthentications.yaml index 9c38fa2ada4..dd83b13cf61 100644 --- a/config/crd/bases/keda.sh_triggerauthentications.yaml +++ b/config/crd/bases/keda.sh_triggerauthentications.yaml @@ -182,6 +182,8 @@ spec: type: string parameter: type: string + secretKey: + type: string versionId: type: string versionStage: diff --git a/pkg/scaling/resolver/aws_secretmanager_handler.go b/pkg/scaling/resolver/aws_secretmanager_handler.go index 595ffb4c6f1..789734a2a82 100644 --- a/pkg/scaling/resolver/aws_secretmanager_handler.go +++ b/pkg/scaling/resolver/aws_secretmanager_handler.go @@ -18,6 +18,7 @@ package resolver import ( "context" + "encoding/json" "fmt" "github.com/aws/aws-sdk-go-v2/aws" @@ -43,9 +44,9 @@ func NewAwsSecretManagerHandler(a *kedav1alpha1.AwsSecretManager) *AwsSecretMana } } -// Read fetches the secret value from AWS Secret Manager using the provided secret name, version ID(optional), and version stage(optional). +// Read fetches the secret value from AWS Secret Manager using the provided secret name, version ID(optional), version stage(optional), and secretKey(optional). // It returns the secret value as a string. -func (ash *AwsSecretManagerHandler) Read(ctx context.Context, logger logr.Logger, secretName, versionID, versionStage string) (string, error) { +func (ash *AwsSecretManagerHandler) Read(ctx context.Context, logger logr.Logger, secretName, versionID, versionStage string, secretKey string) (string, error) { input := &secretsmanager.GetSecretValueInput{ SecretId: aws.String(secretName), } @@ -60,6 +61,27 @@ func (ash *AwsSecretManagerHandler) Read(ctx context.Context, logger logr.Logger logger.Error(err, "Error getting credentials") return "", err } + if secretKey != "" { + // Parse the secret string as JSON + var secretMap map[string]interface{} + err = json.Unmarshal([]byte(*result.SecretString), &secretMap) + if err != nil { + logger.Error(err, "Error parsing secret string as JSON") + return "", err + } + + // Check if the specified secret key exists + if val, ok := secretMap[secretKey]; ok { + // Convert the value to a string and return it + if strVal, isString := val.(string); isString { + return strVal, nil + } + logger.Error(nil, "SecretKey value is not a string") + return "", fmt.Errorf("SecretKey value is not a string") + } + logger.Error(nil, "SecretKey Not Found") + return "", fmt.Errorf("SecretKey Not Found") + } return *result.SecretString, nil } diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index 6ca40672fc6..0032787feb6 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -333,10 +333,10 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge logger.Error(err, "error authenticating to Aws Secret Manager", "triggerAuthRef.Name", triggerAuthRef.Name) } else { for _, secret := range triggerAuthSpec.AwsSecretManager.Secrets { - res, err := awsSecretManagerHandler.Read(ctx, logger, secret.Name, secret.VersionID, secret.VersionStage) + res, err := awsSecretManagerHandler.Read(ctx, logger, secret.Name, secret.VersionID, secret.VersionStage, secret.SecretKey) if err != nil { logger.Error(err, "error trying to read secret from Aws Secret Manager", "triggerAuthRef.Name", triggerAuthRef.Name, - "secret.Name", secret.Name, "secret.Version", secret.VersionID, "secret.VersionStage", secret.VersionStage) + "secret.Name", secret.Name, "secret.Version", secret.VersionID, "secret.VersionStage", secret.VersionStage, "secret.SecretKey", secret.SecretKey) } else { result[secret.Parameter] = res } diff --git a/tests/secret-providers/aws_secretmanager/aws_secretmanager_test.go b/tests/secret-providers/aws_secretmanager/aws_secretmanager_test.go index b185299f4f6..bd6aa29b5d2 100644 --- a/tests/secret-providers/aws_secretmanager/aws_secretmanager_test.go +++ b/tests/secret-providers/aws_secretmanager/aws_secretmanager_test.go @@ -4,19 +4,26 @@ package aws_secret_manager_test import ( + // Standard imports "context" "encoding/base64" + "encoding/json" + "errors" "fmt" "os" "testing" + "time" + // Third-party imports "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" "github.com/joho/godotenv" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" . "github.com/kedacore/keda/v2/tests/helper" @@ -25,6 +32,9 @@ import ( // Load environment variables from .env file var _ = godotenv.Load("../../.env") +// makes sure helper is not removed +var _ = GetRandomNumber() + const ( testName = "aws-secret-manager-test" ) @@ -70,6 +80,7 @@ type templateData struct { SecretManagerSecretName string AwsAccessKeyID string AwsSecretAccessKey string + useJSONSecretFormat bool } const ( @@ -151,6 +162,31 @@ spec: name: {{.SecretManagerSecretName}} ` + triggerAuthenticationSecretKeyTemplate = `apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthenticationName}} + namespace: {{.TestNamespace}} +spec: + awsSecretManager: + credentials: + accessKey: + valueFrom: + secretKeyRef: + name: {{.AwsCredentialsSecretName}} + key: AWS_ACCESS_KEY_ID + accessSecretKey: + valueFrom: + secretKeyRef: + name: {{.AwsCredentialsSecretName}} + key: AWS_SECRET_ACCESS_KEY + region: {{.AwsRegion}} + secrets: + - parameter: connection + name: {{.SecretManagerSecretName}} + secretKey: connectionString +` + scaledObjectTemplate = `apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: @@ -257,11 +293,35 @@ spec: ) func TestAwsSecretManager(t *testing.T) { + // Run the test twice with two different flag values + flags := []bool{true, false} + + for _, useJSONSecretFormat := range flags { + // Define a subtest for each flag value + + t.Run(getTestNameForFlag(useJSONSecretFormat), func(t *testing.T) { + err := AwsSecretManager(t, useJSONSecretFormat) + if err != nil { + t.Errorf("AwsSecretManager(%v) failed: %v", getTestNameForFlag(useJSONSecretFormat), err) + } + }) + } +} + +// Helper to get dynamic test names based on the flag +func getTestNameForFlag(flag bool) string { + if flag { + return "WithFlagTrue" + } + return "WithFlagFalse" +} + +func AwsSecretManager(t *testing.T, useJSONSecretFormat bool) error { require.NotEmpty(t, awsAccessKeyID, "TF_AWS_ACCESS_KEY env variable is required for AWS Secret Manager test") require.NotEmpty(t, awsSecretAccessKey, "TF_AWS_SECRET_KEY env variable is required for AWS Secret Manager test") - // Create the secret in GCP - err := createAWSSecret(t) + // Create the secret in AWS + err := createAWSSecret(t, useJSONSecretFormat) assert.NoErrorf(t, err, "cannot create AWS Secret Manager secret - %s", err) // Create kubernetes resources for PostgreSQL server @@ -280,7 +340,7 @@ func TestAwsSecretManager(t *testing.T) { assert.True(t, ok, "executing a command on PostreSQL Pod should work; Output: %s, ErrorOutput: %s, Error: %s", out, errOut, err) // Create kubernetes resources for testing - data, templates := getTemplateData() + data, templates := getTemplateData(useJSONSecretFormat) KubectlApplyMultipleWithTemplate(t, data, templates) assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), @@ -292,9 +352,10 @@ func TestAwsSecretManager(t *testing.T) { KubectlDeleteMultipleWithTemplate(t, data, templates) DeleteKubernetesResources(t, testNamespace, data, postgreSQLtemplates) - // Delete the secret in GCP + // Delete the secret in AWS err = deleteAWSSecret(t) assert.NoErrorf(t, err, "cannot delete AWS Secret Manager secret - %s", err) + return nil } var data = templateData{ @@ -315,6 +376,7 @@ var data = templateData{ AwsSecretAccessKey: base64.StdEncoding.EncodeToString([]byte(awsSecretAccessKey)), AwsRegion: awsRegion, AwsCredentialsSecretName: awsCredentialsSecretName, + useJSONSecretFormat: false, } func getPostgreSQLTemplateData() (templateData, []Template) { @@ -324,12 +386,19 @@ func getPostgreSQLTemplateData() (templateData, []Template) { } } -func getTemplateData() (templateData, []Template) { +func getTemplateData(useJSONFormat bool) (templateData, []Template) { + var triggerConfig string + if useJSONFormat { + triggerConfig = triggerAuthenticationSecretKeyTemplate + } else { + triggerConfig = triggerAuthenticationTemplate + } + return data, []Template{ {Name: "secretTemplate", Config: secretTemplate}, {Name: "awsCredentialsSecretTemplate", Config: awsCredentialsSecretTemplate}, {Name: "deploymentTemplate", Config: deploymentTemplate}, - {Name: "triggerAuthenticationTemplate", Config: triggerAuthenticationTemplate}, + {Name: "triggerAuthenticationTemplate", Config: triggerConfig}, {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, } } @@ -342,7 +411,7 @@ func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) { "replica count should be %d after 3 minutes", maxReplicaCount) } -func createAWSSecret(t *testing.T) error { +func createAWSSecret(t *testing.T, useJSONFormat bool) error { ctx := context.Background() // Create AWS configuration @@ -360,7 +429,22 @@ func createAWSSecret(t *testing.T) error { client := secretsmanager.NewFromConfig(cfg) // Create the secret value - secretString := postgreSQLConnectionString + var secretString string + if useJSONFormat { + secretObject := map[string]string{ + "connectionString": postgreSQLConnectionString, + } + // Convert the map to a JSON string + jsonData, err := json.Marshal(secretObject) + if err != nil { + return fmt.Errorf("Error converting to JSON: %w", err) + } + + // Print the JSON string + secretString = string(jsonData) + } else { + secretString = postgreSQLConnectionString + } _, err = client.CreateSecret(ctx, &secretsmanager.CreateSecretInput{ Name: &secretManagerSecretName, SecretString: &secretString, @@ -399,7 +483,29 @@ func deleteAWSSecret(t *testing.T) error { if err != nil { return fmt.Errorf("failed to delete AWS Secret Manager secret: %w", err) } + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 300*time.Second, true, func(ctx context.Context) (done bool, err error) { + _, err = client.DescribeSecret(ctx, &secretsmanager.DescribeSecretInput{ + SecretId: &secretManagerSecretName, + }) + if err != nil { + var notFoundErr *types.ResourceNotFoundException + if errors.As(err, ¬FoundErr) { + // Secret successfully deleted + return true, nil + } + // Unexpected error + return false, err + } + // If the secret still exists + return false, nil + }) + if err != nil { + return fmt.Errorf("failed to verify AWS Secret Manager secret deletion: %w", err) + } t.Log("Deleted secret from AWS Secret Manager.") return nil diff --git a/tests/secret-providers/aws_secretmanager_pod_identity/aws_secretmanager_pod_identity_test.go b/tests/secret-providers/aws_secretmanager_pod_identity/aws_secretmanager_pod_identity_test.go index 290b9d07f8b..7ab2bd15542 100644 --- a/tests/secret-providers/aws_secretmanager_pod_identity/aws_secretmanager_pod_identity_test.go +++ b/tests/secret-providers/aws_secretmanager_pod_identity/aws_secretmanager_pod_identity_test.go @@ -4,19 +4,26 @@ package aws_secretmanager_pod_identity_test import ( + // Standard imports "context" "encoding/base64" + "encoding/json" + "errors" "fmt" "os" "testing" + "time" + // Third-party imports "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" "github.com/joho/godotenv" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" . "github.com/kedacore/keda/v2/tests/helper" @@ -25,6 +32,9 @@ import ( // Load environment variables from .env file var _ = godotenv.Load("../../.env") +// makes sure helper is not removed +var _ = GetRandomNumber() + const ( testName = "aws-secret-manage-pod-identity-test" ) @@ -70,6 +80,7 @@ type templateData struct { SecretManagerSecretName string AwsAccessKeyID string AwsSecretAccessKey string + useJSONSecretFormat bool } const ( @@ -141,6 +152,31 @@ spec: name: {{.SecretManagerSecretName}} ` + triggerAuthenticationSecretKeyTemplate = `apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthenticationName}} + namespace: {{.TestNamespace}} +spec: + awsSecretManager: + credentials: + accessKey: + valueFrom: + secretKeyRef: + name: {{.AwsCredentialsSecretName}} + key: AWS_ACCESS_KEY_ID + accessSecretKey: + valueFrom: + secretKeyRef: + name: {{.AwsCredentialsSecretName}} + key: AWS_SECRET_ACCESS_KEY + region: {{.AwsRegion}} + secrets: + - parameter: connection + name: {{.SecretManagerSecretName}} + secretKey: connectionString +` + scaledObjectTemplate = `apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: @@ -247,11 +283,34 @@ spec: ) func TestAwsSecretManager(t *testing.T) { + // Run the test twice with two different flag values + flags := []bool{true, false} + + for _, useJSONSecretFormat := range flags { + // Define a subtest for each flag value + + t.Run(getTestNameForFlag(useJSONSecretFormat), func(t *testing.T) { + err := AwsSecretManager(t, useJSONSecretFormat) + if err != nil { + t.Errorf("AwsSecretManager(%v) failed: %v", getTestNameForFlag(useJSONSecretFormat), err) + } + }) + } +} + +// Helper to get dynamic test names based on the flag +func getTestNameForFlag(flag bool) string { + if flag { + return "WithFlagTrue" + } + return "WithFlagFalse" +} +func AwsSecretManager(t *testing.T, useJSONSecretFormat bool) error { require.NotEmpty(t, awsAccessKeyID, "TF_AWS_ACCESS_KEY env variable is required for AWS Secret Manager test") require.NotEmpty(t, awsSecretAccessKey, "TF_AWS_SECRET_KEY env variable is required for AWS Secret Manager test") - // Create the secret in GCP - err := createAWSSecret(t) + // Create the secret in AWS + err := createAWSSecret(t, useJSONSecretFormat) assert.NoErrorf(t, err, "cannot create AWS Secret Manager secret - %s", err) // Create kubernetes resources for PostgreSQL server @@ -270,7 +329,7 @@ func TestAwsSecretManager(t *testing.T) { assert.True(t, ok, "executing a command on PostreSQL Pod should work; Output: %s, ErrorOutput: %s, Error: %s", out, errOut, err) // Create kubernetes resources for testing - data, templates := getTemplateData() + data, templates := getTemplateData(useJSONSecretFormat) KubectlApplyMultipleWithTemplate(t, data, templates) assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), @@ -282,9 +341,10 @@ func TestAwsSecretManager(t *testing.T) { KubectlDeleteMultipleWithTemplate(t, data, templates) DeleteKubernetesResources(t, testNamespace, data, postgreSQLtemplates) - // Delete the secret in GCP + // Delete the secret in AWS err = deleteAWSSecret(t) assert.NoErrorf(t, err, "cannot delete AWS Secret Manager secret - %s", err) + return nil } var data = templateData{ @@ -305,6 +365,7 @@ var data = templateData{ AwsSecretAccessKey: base64.StdEncoding.EncodeToString([]byte(awsSecretAccessKey)), AwsRegion: awsRegion, AwsCredentialsSecretName: awsCredentialsSecretName, + useJSONSecretFormat: false, } func getPostgreSQLTemplateData() (templateData, []Template) { @@ -314,12 +375,19 @@ func getPostgreSQLTemplateData() (templateData, []Template) { } } -func getTemplateData() (templateData, []Template) { +func getTemplateData(useJSONFormat bool) (templateData, []Template) { + var triggerConfig string + if useJSONFormat { + triggerConfig = triggerAuthenticationSecretKeyTemplate + } else { + triggerConfig = triggerAuthenticationTemplate + } + return data, []Template{ {Name: "secretTemplate", Config: secretTemplate}, {Name: "awsCredentialsSecretTemplate", Config: awsCredentialsSecretTemplate}, {Name: "deploymentTemplate", Config: deploymentTemplate}, - {Name: "triggerAuthenticationTemplate", Config: triggerAuthenticationTemplate}, + {Name: "triggerAuthenticationTemplate", Config: triggerConfig}, {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, } } @@ -332,7 +400,7 @@ func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) { "replica count should be %d after 3 minutes", maxReplicaCount) } -func createAWSSecret(t *testing.T) error { +func createAWSSecret(t *testing.T, useJSONFormat bool) error { ctx := context.Background() // Create AWS configuration @@ -350,7 +418,22 @@ func createAWSSecret(t *testing.T) error { client := secretsmanager.NewFromConfig(cfg) // Create the secret value - secretString := postgreSQLConnectionString + var secretString string + if useJSONFormat { + secretObject := map[string]string{ + "connectionString": postgreSQLConnectionString, + } + // Convert the map to a JSON string + jsonData, err := json.Marshal(secretObject) + if err != nil { + return fmt.Errorf("Error converting to JSON: %w", err) + } + + // Print the JSON string + secretString = string(jsonData) + } else { + secretString = postgreSQLConnectionString + } _, err = client.CreateSecret(ctx, &secretsmanager.CreateSecretInput{ Name: &secretManagerSecretName, SecretString: &secretString, @@ -390,6 +473,36 @@ func deleteAWSSecret(t *testing.T) error { return fmt.Errorf("failed to delete AWS Secret Manager secret: %w", err) } + // Wait for the delete of the secret to really take effect + + if err != nil { + return fmt.Errorf("failed to verify AWS Secret Manager secret deletion: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 300*time.Second, true, func(ctx context.Context) (done bool, err error) { + _, err = client.DescribeSecret(ctx, &secretsmanager.DescribeSecretInput{ + SecretId: &secretManagerSecretName, + }) + if err != nil { + var notFoundErr *types.ResourceNotFoundException + if errors.As(err, ¬FoundErr) { + // Secret successfully deleted + return true, nil + } + // Unexpected error + return false, err + } + // If the secret still exists + return false, nil + }) + + if err != nil { + return fmt.Errorf("failed to verify AWS Secret Manager secret deletion: %w", err) + } + t.Log("Deleted secret from AWS Secret Manager.") return nil