feat: rework aws auth (#1379)

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/alitto/pond v1.9.0
 	github.com/andygrunwald/go-jira v1.16.0
 	github.com/aws/aws-sdk-go v1.49.10
-	github.com/aws/aws-sdk-go-v2 v1.30.1
+	github.com/aws/aws-sdk-go-v2 v1.30.3
 	github.com/aws/aws-sdk-go-v2/config v1.27.23
 	github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.0
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.167.1
@@ -45,7 +45,7 @@ require (
 	github.com/jackc/pgconn v1.14.3
 	github.com/jackc/pgtype v1.14.0
 	github.com/jackc/pgx/v4 v4.18.3
-	github.com/kaytu-io/kaytu-aws-describer v0.55.6
+	github.com/kaytu-io/kaytu-aws-describer v0.56.0
 	github.com/kaytu-io/kaytu-azure-describer v0.35.6
 	github.com/kaytu-io/kaytu-util v0.0.0-20240729100022-e4d357e0ebf0
 	github.com/kaytu-io/pennywise v1.7.16
@@ -227,8 +227,8 @@ require (
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.23 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect
 	github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.26.5 // indirect
@@ -306,6 +306,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/kinesisanalyticsv2 v1.21.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/kinesisvideo v1.21.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/kms v1.27.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/lakeformation v1.35.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/lightsail v1.32.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/macie2 v1.34.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/mediastore v1.18.5 // indirect

go.sum

@@ -579,8 +579,8 @@ github.com/aws/aws-sdk-go v1.49.10 h1:xcTIazQPKoQWmegkQu5C7oPDgXwGaN7/E9y6TGmxNU
 github.com/aws/aws-sdk-go v1.49.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
-github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o=
-github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
+github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
+github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
 github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4=
@@ -593,11 +593,11 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEu
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
@@ -762,6 +762,8 @@ github.com/aws/aws-sdk-go-v2/service/kinesisvideo v1.21.5 h1:3BVBavJrQrrv8K1McEJ
 github.com/aws/aws-sdk-go-v2/service/kinesisvideo v1.21.5/go.mod h1:6UColZloou6uAkpoJDqgzhd0nKvq5H0BVuXar+OxnDY=
 github.com/aws/aws-sdk-go-v2/service/kms v1.27.9 h1:W9PbZAZAEcelhhjb7KuwUtf+Lbc+i7ByYJRuWLlnxyQ=
 github.com/aws/aws-sdk-go-v2/service/kms v1.27.9/go.mod h1:2tFmR7fQnOdQlM2ZCEPpFnBIQD1U8wmXmduBgZbOag0=
+github.com/aws/aws-sdk-go-v2/service/lakeformation v1.35.3 h1:Rfl7JjXVdriUprd8TTlbgcTyPU/Pl+v/O/nMD9HYpgA=
+github.com/aws/aws-sdk-go-v2/service/lakeformation v1.35.3/go.mod h1:cyogDr92z2UF8fBoRN/+/gKuVTrxBD10bo6PVn3tDeQ=
 github.com/aws/aws-sdk-go-v2/service/lambda v1.49.5 h1:ZHVbzOnoj5nXxUug8iWzqg2Tmp6Jc4CE5tPfoE96qrs=
 github.com/aws/aws-sdk-go-v2/service/lambda v1.49.5/go.mod h1:0V5z1X/8NA9eQ5cZSz5ZaHU8xA/hId2ZAlsHeO7Jrdk=
 github.com/aws/aws-sdk-go-v2/service/lightsail v1.32.5 h1:0KVnA62WGcVdeJKH+DTUkxNms2OsIky+AmB2iX93eAs=
@@ -1785,8 +1787,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
-github.com/kaytu-io/kaytu-aws-describer v0.55.6 h1:fSz1KUSr/DvPraRP98z3WKUTaYucE5i+5VMoIOkkK3I=
-github.com/kaytu-io/kaytu-aws-describer v0.55.6/go.mod h1:98W5RVmAUrkPgJmPh12YpLoW5vq+hDIDXBcVXUlu0V4=
+github.com/kaytu-io/kaytu-aws-describer v0.56.0 h1:d29HND7v6CQj/nrb+DQbgKM4pPzYzV7DlEhFsLHfidM=
+github.com/kaytu-io/kaytu-aws-describer v0.56.0/go.mod h1:q/TWtMkG8OJFHhhQlHvkgQBTOaisbz+7oWz8iuZg4no=
 github.com/kaytu-io/kaytu-azure-describer v0.35.6 h1:VYyzeblkTNuBwrPVAPLz6jBMqoT4qn+Hd794/jM9jbk=
 github.com/kaytu-io/kaytu-azure-describer v0.35.6/go.mod h1:B3d980pSloy3FzCKouf+ql9+IjS/Ntzs/xFOC/FOiIo=
 github.com/kaytu-io/kaytu-util v0.0.0-20240729100022-e4d357e0ebf0 h1:9H0Hnv9yt41uhiCowx2swIR03mKmSEZpuNgRg1Hfcjc=

pkg/onboard/api/source.go

@@ -21,14 +21,13 @@ const (
 )
 
 type AWSCredentialConfig struct {
-	AccountId            string   `json:"accountId"`
-	Regions              []string `json:"regions,omitempty"`
-	AccessKey            string   `json:"accessKey" validate:"required"`
-	SecretKey            string   `json:"secretKey" validate:"required"`
-	AssumeRoleName       string   `json:"assumeRoleName,omitempty"`
-	AssumeAdminRoleName  string   `json:"assumeAdminRoleName,omitempty"`
-	AssumeRolePolicyName string   `json:"assumeRolePolicyName,omitempty"`
-	ExternalId           *string  `json:"externalId,omitempty"`
+	AccountId           string   `json:"accountId"`
+	Regions             []string `json:"regions,omitempty"`
+	AccessKey           string   `json:"accessKey" validate:"required"`
+	SecretKey           string   `json:"secretKey" validate:"required"`
+	AssumeRoleName      string   `json:"assumeRoleName,omitempty"`
+	AssumeAdminRoleName string   `json:"assumeAdminRoleName,omitempty"`
+	ExternalId          *string  `json:"externalId,omitempty"`
 }
 
 func (s AWSCredentialConfig) AsMap() map[string]any {

pkg/onboard/api/v2/credential.go

@@ -29,10 +29,12 @@ func (req CreateCredentialV2Request) GetAWSConfig() (*AWSCredentialV2Config, err
 }
 
 type AWSCredentialV2Config struct {
-	AccountID           string   `json:"accountID"`
-	AssumeRoleName      string   `json:"assumeRoleName"`
-	HealthCheckPolicies []string `json:"healthCheckPolicies"`
-	ExternalId          *string  `json:"externalId"`
+	AccountID      string  `json:"accountID"`
+	AssumeRoleName string  `json:"assumeRoleName"`
+	ExternalId     *string `json:"externalId,omitempty"`
+
+	AccessKey *string `json:"accessKey,omitempty"`
+	SecretKey *string `json:"secretKey,omitempty"`
 }
 
 func (s AWSCredentialV2Config) AsMap() map[string]any {

pkg/onboard/compatibility.go

@@ -18,15 +18,23 @@ func (h HttpHandler) CredentialV2ToV1(ctx context.Context, newCred model.Credent
 		return "", err
 	}
 
+	aKey := h.masterAccessKey
+	sKey := h.masterSecretKey
+	if awsCnf.AccessKey != nil {
+		aKey = *awsCnf.AccessKey
+	}
+	if awsCnf.SecretKey != nil {
+		sKey = *awsCnf.SecretKey
+	}
+
 	newConf := api.AWSCredentialConfig{
-		AccountId:            awsCnf.AccountID,
-		Regions:              nil,
-		AccessKey:            h.masterAccessKey,
-		SecretKey:            h.masterSecretKey,
-		AssumeRoleName:       awsCnf.AssumeRoleName,
-		AssumeAdminRoleName:  awsCnf.AssumeRoleName,
-		AssumeRolePolicyName: "",
-		ExternalId:           awsCnf.ExternalId,
+		AccountId:           awsCnf.AccountID,
+		Regions:             nil,
+		AccessKey:           aKey,
+		SecretKey:           sKey,
+		AssumeRoleName:      awsCnf.AssumeRoleName,
+		AssumeAdminRoleName: awsCnf.AssumeRoleName,
+		ExternalId:          awsCnf.ExternalId,
 	}
 
 	newSecret, err := h.vaultSc.Encrypt(ctx, newConf.AsMap())

pkg/onboard/handler_connection.go

@@ -37,8 +37,18 @@ func (h HttpHandler) checkConnectionHealth(ctx context.Context, connection model
 				h.logger.Error("failed to get aws config", zap.Error(err), zap.String("sourceId", connection.SourceId))
 				return connection, err
 			}
+
+			aKey := h.masterAccessKey
+			sKey := h.masterSecretKey
+			if awsCnf.AccessKey != nil {
+				aKey = *awsCnf.AccessKey
+			}
+			if awsCnf.SecretKey != nil {
+				sKey = *awsCnf.SecretKey
+			}
+
 			assumeRoleArn := kaytuAws.GetRoleArnFromName(connection.SourceId, awsCnf.AssumeRoleName)
-			sdkCnf, err := kaytuAws.GetConfig(ctx, h.masterAccessKey, h.masterSecretKey, "", assumeRoleArn, awsCnf.ExternalId)
+			sdkCnf, err := kaytuAws.GetConfig(ctx, aKey, sKey, "", assumeRoleArn, awsCnf.ExternalId)
 			if err != nil {
 				h.logger.Error("failed to get aws config", zap.Error(err), zap.String("sourceId", connection.SourceId))
 				return connection, err
@@ -60,23 +70,7 @@ func (h HttpHandler) checkConnectionHealth(ctx context.Context, connection model
 			}
 
 			assetDiscoveryAttached = true
-			awsAssetDiscovery, err := h.metadataClient.GetConfigMetadata(&httpclient.Context{UserRole: api.InternalRole}, models.MetadataKeyAssetDiscoveryAWSPolicyARNs)
-			if err != nil {
-				return connection, err
-			}
-
-			for _, policyARN := range strings.Split(awsAssetDiscovery.GetValue().(string), ",") {
-				policyARN = strings.ReplaceAll(policyARN, "${accountID}", connection.SourceId)
-				if !utils.Includes(policyARNs, policyARN) {
-					h.logger.Error("policy is not there", zap.String("policyARN", policyARN), zap.Strings("attachedPolicies", policyARNs))
-					assetDiscoveryAttached = false
-				}
-			}
-
 			spendAttached = connection.Credential.SpendDiscovery != nil && *connection.Credential.SpendDiscovery
-
-			//TODO
-
 		} else {
 			var awsCnf describe.AWSAccountConfig
 			awsCnf, err = describe.AWSAccountConfigFromMap(cnf)
@@ -95,12 +89,8 @@ func (h HttpHandler) checkConnectionHealth(ctx context.Context, connection model
 				h.logger.Error("failed to get aws config", zap.Error(err), zap.String("sourceId", connection.SourceId))
 				return connection, err
 			}
-			if awsCnf.AccountID != connection.SourceId {
-				assetDiscoveryAttached, err = kaytuAws.CheckAttachedPolicy(h.logger, sdkCnf, awsCnf.AssumeRoleName, kaytuAws.GetPolicyArnFromName(connection.SourceId, awsCnf.AssumeRolePolicyName))
-			} else {
-				assetDiscoveryAttached, err = kaytuAws.CheckAttachedPolicy(h.logger, sdkCnf, "", kaytuAws.GetPolicyArnFromName(connection.SourceId, awsCnf.AssumeRolePolicyName))
-			}
-			spendAttached = assetDiscoveryAttached // backward compatibility
+			assetDiscoveryAttached = true
+			spendAttached = connection.Credential.SpendDiscovery != nil && *connection.Credential.SpendDiscovery
 			if err == nil && assetDiscoveryAttached && updateMetadata {
 				if sdkCnf.Region == "" {
 					sdkCnf.Region = "us-east-1"

pkg/onboard/handler_credential.go

@@ -12,33 +12,38 @@ import (
 	"github.com/kaytu-io/kaytu-aws-describer/aws/describer"
 	kaytuAzure "github.com/kaytu-io/kaytu-azure-describer/azure"
 	"github.com/kaytu-io/kaytu-engine/pkg/describe"
-	"github.com/kaytu-io/kaytu-engine/pkg/metadata/models"
 	"github.com/kaytu-io/kaytu-engine/pkg/onboard/api"
 	apiv2 "github.com/kaytu-io/kaytu-engine/pkg/onboard/api/v2"
 	"github.com/kaytu-io/kaytu-engine/pkg/utils"
 	"github.com/kaytu-io/kaytu-engine/services/integration/model"
-	api2 "github.com/kaytu-io/kaytu-util/pkg/api"
-	"github.com/kaytu-io/kaytu-util/pkg/httpclient"
 	"github.com/kaytu-io/kaytu-util/pkg/source"
 	"github.com/labstack/echo/v4"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
 	"net/http"
-	"strings"
 	"time"
 )
 
 func generateRoleARN(accountID, roleName string) string {
 	return fmt.Sprintf("arn:aws:iam::%s:role/%s", accountID, roleName)
 }
 
-func (h HttpHandler) GetAWSSDKConfig(ctx context.Context, roleARN string, externalID *string) (aws.Config, error) {
+func (h HttpHandler) GetAWSSDKConfig(ctx context.Context, roleARN string, accessKey, secretKey, externalID *string) (aws.Config, error) {
+	aKey := h.masterAccessKey
+	sKey := h.masterSecretKey
+	if accessKey != nil {
+		aKey = *accessKey
+	}
+	if secretKey != nil {
+		sKey = *secretKey
+	}
+
 	awsConfig, err := kaytuAws.GetConfig(
 		ctx,
-		h.masterAccessKey,
-		h.masterSecretKey,
+		aKey,
+		sKey,
 		"",
 		roleARN,
 		externalID,
@@ -90,7 +95,7 @@ func (h HttpHandler) ExtractCredentialMetadata(accountID string, org *awsOrgType
 }
 
 func (h HttpHandler) createAWSCredential(ctx context.Context, req apiv2.CreateCredentialV2Request) (*apiv2.CreateCredentialV2Response, error) {
-	awsConfig, err := h.GetAWSSDKConfig(ctx, generateRoleARN(req.AWSConfig.AccountID, req.AWSConfig.AssumeRoleName), req.AWSConfig.ExternalId)
+	awsConfig, err := h.GetAWSSDKConfig(ctx, generateRoleARN(req.AWSConfig.AccountID, req.AWSConfig.AssumeRoleName), req.AWSConfig.AccessKey, req.AWSConfig.SecretKey, req.AWSConfig.ExternalId)
 	if err != nil {
 		return nil, err
 	}
@@ -144,10 +149,19 @@ func (h HttpHandler) autoOnboardAWSAccountsV2(ctx context.Context, credential mo
 		return nil, err
 	}
 
+	aKey := h.masterAccessKey
+	sKey := h.masterSecretKey
+	if awsCnf.AccessKey != nil {
+		aKey = *awsCnf.AccessKey
+	}
+	if awsCnf.SecretKey != nil {
+		sKey = *awsCnf.SecretKey
+	}
+
 	awsConfig, err := kaytuAws.GetConfig(
 		ctx,
-		h.masterAccessKey,
-		h.masterSecretKey,
+		aKey,
+		sKey,
 		"",
 		generateRoleARN(awsCnf.AccountID, awsCnf.AssumeRoleName),
 		awsCnf.ExternalId,
@@ -317,7 +331,7 @@ func (h HttpHandler) checkCredentialHealthV2(ctx context.Context, cred model.Cre
 			h.logger.Error("failed to parse aws config", zap.Error(err))
 			return false, err
 		}
-		sdkCnf, err := h.GetAWSSDKConfig(ctx, generateRoleARN(awsConfig.AccountID, awsConfig.AssumeRoleName), awsConfig.ExternalId)
+		sdkCnf, err := h.GetAWSSDKConfig(ctx, generateRoleARN(awsConfig.AccountID, awsConfig.AssumeRoleName), awsConfig.AccessKey, awsConfig.SecretKey, awsConfig.ExternalId)
 		if err != nil {
 			h.logger.Error("failed to get aws sdk config", zap.Error(err))
 			return false, err
@@ -358,20 +372,6 @@ func (h HttpHandler) checkCredentialHealthV2(ctx context.Context, cred model.Cre
 		}
 
 		spendAttached := true
-		ctx2 := &httpclient.Context{UserRole: api2.InternalRole}
-		ctx2.Ctx = ctx
-		awsSpendDiscovery, err := h.metadataClient.GetConfigMetadata(ctx2, models.MetadataKeySpendDiscoveryAWSPolicyARNs)
-		if err != nil {
-			h.logger.Error("failed to get spend discovery aws policy arns", zap.Error(err))
-			return false, err
-		}
-		for _, policyARN := range strings.Split(awsSpendDiscovery.GetValue().(string), ",") {
-			policyARN = strings.ReplaceAll(policyARN, "${accountID}", awsConfig.AccountID)
-			if !utils.Includes(policyARNs, policyARN) {
-				h.logger.Error("policy is not there", zap.String("policyARN", policyARN), zap.Strings("attachedPolicies", policyARNs))
-				spendAttached = false
-			}
-		}
 		cred.SpendDiscovery = &spendAttached
 	default:
 		return false, errors.New("not implemented")