diff --git a/API.md b/API.md index 82181fa3..d222f48f 100644 --- a/API.md +++ b/API.md @@ -10997,18 +10997,21 @@ const s3Catalog: S3Catalog = { ... } | **Name** | **Type** | **Description** | | --- | --- | --- | -| database | @aws-cdk/aws-glue-alpha.Database | *No description.* | -| tablePrefix | string | *No description.* | +| database | @aws-cdk/aws-glue-alpha.IDatabase | The AWS Glue database that will contain the tables created when the flow executes. | +| tablePrefix | string | The prefix for the tables created in the AWS Glue database. | +| role | aws-cdk-lib.aws_iam.IRole | The IAM Role that will be used for data catalog operations. | --- ##### `database`Required ```typescript -public readonly database: Database; +public readonly database: IDatabase; ``` -- *Type:* @aws-cdk/aws-glue-alpha.Database +- *Type:* @aws-cdk/aws-glue-alpha.IDatabase + +The AWS Glue database that will contain the tables created when the flow executes. --- @@ -11020,6 +11023,21 @@ public readonly tablePrefix: string; - *Type:* string +The prefix for the tables created in the AWS Glue database. + +--- + +##### `role`Optional + +```typescript +public readonly role: IRole; +``` + +- *Type:* aws-cdk-lib.aws_iam.IRole +- *Default:* A new role will be created + +The IAM Role that will be used for data catalog operations. + --- ### S3DestinationProps @@ -11036,9 +11054,9 @@ const s3DestinationProps: S3DestinationProps = { ... } | **Name** | **Type** | **Description** | | --- | --- | --- | -| location | S3Location | *No description.* | -| catalog | S3Catalog | *No description.* | -| formatting | S3OutputFormatting | *No description.* | +| location | S3Location | The S3 location of the files with the retrieved data. | +| catalog | S3Catalog | The AWS Glue cataloging options. | +| formatting | S3OutputFormatting | The formatting options for the output files. | --- @@ -11050,6 +11068,8 @@ public readonly location: S3Location; - *Type:* S3Location +The S3 location of the files with the retrieved data. + --- ##### `catalog`Optional @@ -11060,6 +11080,8 @@ public readonly catalog: S3Catalog; - *Type:* S3Catalog +The AWS Glue cataloging options. + --- ##### `formatting`Optional @@ -11070,6 +11092,8 @@ public readonly formatting: S3OutputFormatting; - *Type:* S3OutputFormatting +The formatting options for the output files. + --- ### S3FileAggregation @@ -11244,10 +11268,10 @@ const s3OutputFormatting: S3OutputFormatting = { ... } | **Name** | **Type** | **Description** | | --- | --- | --- | -| aggregation | S3FileAggregation | *No description.* | -| filePrefix | S3OutputFilePrefix | *No description.* | -| fileType | S3OutputFileType | *No description.* | -| preserveSourceDataTypes | boolean | *No description.* | +| aggregation | S3FileAggregation | Sets an aggregation approach per flow run. | +| filePrefix | S3OutputFilePrefix | Sets a prefix approach for files generated during a flow execution. | +| fileType | S3OutputFileType | Sets the file type for the output files. | +| preserveSourceDataTypes | boolean | Specifies whether AppFlow should attempt data type mapping from source when the destination output file type is Parquet. | --- @@ -11259,6 +11283,8 @@ public readonly aggregation: S3FileAggregation; - *Type:* S3FileAggregation +Sets an aggregation approach per flow run. + --- ##### `filePrefix`Optional @@ -11269,6 +11295,8 @@ public readonly filePrefix: S3OutputFilePrefix; - *Type:* S3OutputFilePrefix +Sets a prefix approach for files generated during a flow execution. + --- ##### `fileType`Optional @@ -11278,6 +11306,9 @@ public readonly fileType: S3OutputFileType; ``` - *Type:* S3OutputFileType +- *Default:* JSON file type + +Sets the file type for the output files. --- @@ -11288,6 +11319,9 @@ public readonly preserveSourceDataTypes: boolean; ``` - *Type:* boolean +- *Default:* do not preserve source data files + +Specifies whether AppFlow should attempt data type mapping from source when the destination output file type is Parquet. --- @@ -18130,28 +18164,36 @@ The file type that Amazon AppFlow gets from your Amazon S3 bucket. ### S3OutputFileType +Output file type supported by Amazon S3 Destination connector. + #### Members | **Name** | **Description** | | --- | --- | -| CSV | *No description.* | -| JSON | *No description.* | -| PARQUET | *No description.* | +| CSV | CSV file type. | +| JSON | JSON file type. | +| PARQUET | Parquet file type. | --- ##### `CSV` +CSV file type. + --- ##### `JSON` +JSON file type. + --- ##### `PARQUET` +Parquet file type. + --- diff --git a/src/s3/destination.ts b/src/s3/destination.ts index e63116fa..c8e344b7 100644 --- a/src/s3/destination.ts +++ b/src/s3/destination.ts @@ -2,9 +2,16 @@ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 */ -import { Database } from '@aws-cdk/aws-glue-alpha'; +import { IDatabase } from '@aws-cdk/aws-glue-alpha'; +import { Stack } from 'aws-cdk-lib'; import { CfnFlow } from 'aws-cdk-lib/aws-appflow'; -import { Effect, Policy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; +import { + Effect, + IRole, + PolicyStatement, + Role, + ServicePrincipal, +} from 'aws-cdk-lib/aws-iam'; import { IConstruct } from 'constructs'; import { S3ConnectorType } from './type'; import { AppFlowPermissionsManager } from '../core/appflow-permissions-manager'; @@ -19,35 +26,62 @@ export interface S3FileAggregation { * The maximum size, in MB, of the file containing portion of incoming data */ readonly fileSize?: number; -}; +} export interface S3OutputFilePrefix { readonly hierarchy?: S3OutputFilePrefixHierarchy[]; readonly format?: S3OutputFilePrefixFormat; readonly type?: S3OutputFilePrefixType; -}; +} export interface S3OutputFormatting { + /** + * Sets an aggregation approach per flow run + */ readonly aggregation?: S3FileAggregation; + + /** + * Sets the file type for the output files + * @default - JSON file type + */ readonly fileType?: S3OutputFileType; + /** + * Sets a prefix approach for files generated during a flow execution + */ readonly filePrefix?: S3OutputFilePrefix; + /** + * Specifies whether AppFlow should attempt data type mapping from source when the destination output file type is Parquet + * @default - do not preserve source data files + */ readonly preserveSourceDataTypes?: boolean; } export enum S3OutputAggregationType { NONE = 'None', - SINGLE_FILE = 'SingleFile' + SINGLE_FILE = 'SingleFile', } +/** + * Output file type supported by Amazon S3 Destination connector + */ export enum S3OutputFileType { + /** + * CSV file type + */ CSV = 'CSV', + /** + * JSON file type + */ JSON = 'JSON', - PARQUET = 'PARQUET' + /** + * Parquet file type + */ + PARQUET = 'Parquet', } export enum S3OutputFilePrefixHierarchy { EXECUTION_ID = 'EXECUTION_ID', - SCHEMA_VERSION = 'SCHEMA_VERSION' + SCHEMA_VERSION = 'SCHEMA_VERSION', } export enum S3OutputFilePrefixFormat { @@ -55,67 +89,104 @@ export enum S3OutputFilePrefixFormat { HOUR = 'HOUR', MINUTE = 'MINUTE', MONTH = 'MONTH', - YEAR = 'YEAR' + YEAR = 'YEAR', } export enum S3OutputFilePrefixType { FILENAME = 'FILENAME', PATH = 'PATH', - PATH_AND_FILE = 'PATH_AND_FILE' + PATH_AND_FILE = 'PATH_AND_FILE', } export interface S3Catalog { - readonly database: Database; + /** + * The IAM Role that will be used for data catalog operations + * @default - A new role will be created + */ + readonly role?: IRole; + + /** + * The AWS Glue database that will contain the tables created when the flow executes + */ + readonly database: IDatabase; + + /** + * The prefix for the tables created in the AWS Glue database + */ readonly tablePrefix: string; } export interface S3DestinationProps { + /** + * The S3 location of the files with the retrieved data + */ readonly location: S3Location; + + /** + * The AWS Glue cataloging options + */ readonly catalog?: S3Catalog; + + /** + * The formatting options for the output files + */ readonly formatting?: S3OutputFormatting; } export class S3Destination implements IDestination { - public readonly connectorType: ConnectorType = S3ConnectorType.instance; - private readonly compatibleFlows: FlowType[] = [FlowType.ON_DEMAND, FlowType.SCHEDULED]; + private readonly compatibleFlows: FlowType[] = [ + FlowType.ON_DEMAND, + FlowType.SCHEDULED, + ]; - constructor(private readonly props: S3DestinationProps) { } + constructor(private readonly props: S3DestinationProps) {} private buildAggregationConfig(aggregation?: S3FileAggregation) { - return aggregation && { - aggregationType: aggregation.type, - // TODO: make sure it should use mebibytes - targetFileSize: aggregation.fileSize && aggregation.fileSize, - }; + return ( + aggregation && { + aggregationType: aggregation.type, + // TODO: make sure it should use mebibytes + targetFileSize: aggregation.fileSize && aggregation.fileSize, + } + ); } private buildPrefixConfig(filePrefix?: S3OutputFilePrefix) { - return filePrefix && { - pathPrefixHierarchy: filePrefix.hierarchy, - prefixFormat: filePrefix.format, - prefixType: filePrefix.type, - }; + return ( + filePrefix && { + pathPrefixHierarchy: filePrefix.hierarchy, + prefixFormat: filePrefix.format, + prefixType: filePrefix.type, + } + ); } bind(flow: IFlow): CfnFlow.DestinationFlowConfigProperty { - - // TODO: make sure this even makes sense if (!this.compatibleFlows.includes(flow.type)) { - throw new Error(`Flow of type ${flow.type} does not support S3 as a destination`); + throw new Error( + `Flow of type ${flow.type} does not support S3 as a destination`, + ); } this.tryAddNodeDependency(flow, this.props.location.bucket); - AppFlowPermissionsManager.instance().grantBucketWrite(this.props.location.bucket); - if (this.props.catalog) { + AppFlowPermissionsManager.instance().grantBucketWrite( + this.props.location.bucket, + ); - const { role, policy } = this.buildRoleAndPolicyForCatalog(flow); + if (this.props.catalog) { + const role = + this.props.catalog.role ?? + this.buildRoleAndPolicyForCatalog( + flow, + this.props.catalog.database, + this.props.catalog.tablePrefix, + ); this.tryAddNodeDependency(flow, this.props.catalog.database); this.tryAddNodeDependency(flow, role); - this.tryAddNodeDependency(flow, policy); flow._addCatalog({ glueDataCatalog: { @@ -128,7 +199,8 @@ export class S3Destination implements IDestination { return { connectorType: this.connectorType.asProfileConnectorType, - destinationConnectorProperties: this.buildDestinationConnectorProperties(), + destinationConnectorProperties: + this.buildDestinationConnectorProperties(), }; } @@ -136,54 +208,66 @@ export class S3Destination implements IDestination { * see: https://docs.aws.amazon.com/appflow/latest/userguide/security_iam_id-based-policy-examples.html#security_iam_id-based-policy-examples-access-gdc * see: https://docs.aws.amazon.com/appflow/latest/userguide/security_iam_service-role-policies.html#access-gdc * @param flow + * @param database - the AWS Glue database + * @param tablePrefix - the prefix for the tables that will be created in the AWS Glue database * @returns a tuple consisting of a role for cataloguing with a specified policy */ - private buildRoleAndPolicyForCatalog(flow: IFlow) { + private buildRoleAndPolicyForCatalog( + flow: IFlow, + database: IDatabase, + tablePrefix: string, + ) { const role = new Role(flow.stack, `${flow.node.id}GlueAccessRole`, { assumedBy: new ServicePrincipal('appflow.amazonaws.com'), }); - const policy = new Policy(flow.stack, `${flow.node.id}GlueAccessRolePolicy`, { - roles: [role], - statements: [ - new PolicyStatement({ - effect: Effect.ALLOW, - actions: [ - 'glue:BatchCreatePartition', - 'glue:CreatePartitionIndex', - 'glue:DeleteDatabase', - 'glue:GetTableVersions', - 'glue:GetPartitions', - 'glue:BatchDeletePartition', - 'glue:DeleteTableVersion', - 'glue:UpdateTable', - 'glue:DeleteTable', - 'glue:DeletePartitionIndex', - 'glue:GetTableVersion', - 'glue:CreatePartition', - 'glue:UntagResource', - 'glue:UpdatePartition', - 'glue:TagResource', - 'glue:UpdateDatabase', - 'glue:CreateTable', - 'glue:BatchUpdatePartition', - 'glue:GetTables', - 'glue:BatchGetPartition', - 'glue:GetDatabases', - 'glue:GetPartitionIndexes', - 'glue:GetTable', - 'glue:GetDatabase', - 'glue:GetPartition', - 'glue:CreateDatabase', - 'glue:BatchDeleteTableVersion', - 'glue:BatchDeleteTable', - 'glue:DeletePartition', - ], - resources: ['*'], - }), - ], - }); - return { role, policy }; + // see: https://docs.aws.amazon.com/appflow/latest/userguide/security_iam_id-based-policy-examples.html#security_iam_id-based-policy-examples-access-gdc + role.addToPolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + 'glue:BatchCreatePartition', + 'glue:CreatePartitionIndex', + 'glue:DeleteDatabase', + 'glue:GetTableVersions', + 'glue:GetPartitions', + 'glue:BatchDeletePartition', + 'glue:DeleteTableVersion', + 'glue:UpdateTable', + 'glue:DeleteTable', + 'glue:DeletePartitionIndex', + 'glue:GetTableVersion', + 'glue:CreatePartition', + 'glue:UntagResource', + 'glue:UpdatePartition', + 'glue:TagResource', + 'glue:UpdateDatabase', + 'glue:CreateTable', + 'glue:BatchUpdatePartition', + 'glue:GetTables', + 'glue:BatchGetPartition', + 'glue:GetDatabases', + 'glue:GetPartitionIndexes', + 'glue:GetTable', + 'glue:GetDatabase', + 'glue:GetPartition', + 'glue:CreateDatabase', + 'glue:BatchDeleteTableVersion', + 'glue:BatchDeleteTable', + 'glue:DeletePartition', + ], + resources: [ + database.catalogArn, + database.databaseArn, + Stack.of(flow).formatArn({ + service: 'glue', + resource: 'table', + resourceName: `${database.databaseName}/${tablePrefix}*`, + }), + ], + }), + ); + return role; } private buildDestinationConnectorProperties(): CfnFlow.DestinationConnectorPropertiesProperty { @@ -198,18 +282,26 @@ export class S3Destination implements IDestination { bucketName: bucketName, bucketPrefix: this.props.location.prefix, s3OutputFormatConfig: this.props.formatting && { - aggregationConfig: this.buildAggregationConfig(this.props.formatting.aggregation), + aggregationConfig: this.buildAggregationConfig( + this.props.formatting.aggregation, + ), fileType: this.props.formatting.fileType ?? S3OutputFileType.JSON, - prefixConfig: this.buildPrefixConfig(this.props.formatting.filePrefix), - preserveSourceDataTyping: this.props.formatting.preserveSourceDataTypes, + prefixConfig: this.buildPrefixConfig( + this.props.formatting.filePrefix, + ), + preserveSourceDataTyping: + this.props.formatting.preserveSourceDataTypes, }, }, }; } - private tryAddNodeDependency(scope: IConstruct, resource?: IConstruct | string) { + private tryAddNodeDependency( + scope: IConstruct, + resource?: IConstruct | string, + ) { if (resource && typeof resource !== 'string') { scope.node.addDependency(resource); } } -} \ No newline at end of file +} diff --git a/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.assets.json index 1b58b8dd..de1be1c2 100644 --- a/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.assets.json +++ b/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.assets.json @@ -14,7 +14,7 @@ } } }, - "d7dd40c8f35d1faa379c9b865827bc504b760f92640ac78c67acbb4c6e5db6b7": { + "7b44775df1172ce015e29ddc2f739b0981807203d058a7db35aa0f2c68d41468": { "source": { "path": "TestStack.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "d7dd40c8f35d1faa379c9b865827bc504b760f92640ac78c67acbb4c6e5db6b7.json", + "objectKey": "7b44775df1172ce015e29ddc2f739b0981807203d058a7db35aa0f2c68d41468.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.template.json b/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.template.json index 94d81060..a509b67a 100644 --- a/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.template.json +++ b/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.template.json @@ -380,8 +380,8 @@ } }, "DependsOn": [ + "OnDemandFlowGlueAccessRoleDefaultPolicy5A6D1D10", "OnDemandFlowGlueAccessRole2AB92366", - "OnDemandFlowGlueAccessRolePolicy68F25044", "TestBucketAutoDeleteObjectsCustomResource8FEAABD5", "TestBucketPolicyBA12ED38", "TestBucket560B80BC", @@ -406,7 +406,7 @@ } } }, - "OnDemandFlowGlueAccessRolePolicy68F25044": { + "OnDemandFlowGlueAccessRoleDefaultPolicy5A6D1D10": { "Type": "AWS::IAM::Policy", "Properties": { "PolicyDocument": { @@ -444,12 +444,80 @@ "glue:DeletePartition" ], "Effect": "Allow", - "Resource": "*" + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":catalog" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":database/", + { + "Ref": "TestDatabase7A4A91C2" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "TestDatabase7A4A91C2" + }, + "/test_prefix*" + ] + ] + } + ] } ], "Version": "2012-10-17" }, - "PolicyName": "OnDemandFlowGlueAccessRolePolicy68F25044", + "PolicyName": "OnDemandFlowGlueAccessRoleDefaultPolicy5A6D1D10", "Roles": [ { "Ref": "OnDemandFlowGlueAccessRole2AB92366" diff --git a/test/s3/s3.test.ts b/test/s3/s3.test.ts index a686216d..a4d63029 100644 --- a/test/s3/s3.test.ts +++ b/test/s3/s3.test.ts @@ -2,8 +2,10 @@ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 */ +import { Database } from '@aws-cdk/aws-glue-alpha'; import { Stack } from 'aws-cdk-lib'; -import { Template } from 'aws-cdk-lib/assertions'; +import { Template, Match } from 'aws-cdk-lib/assertions'; +import { Role } from 'aws-cdk-lib/aws-iam'; import { Bucket } from 'aws-cdk-lib/aws-s3'; import { Mapping, @@ -90,7 +92,7 @@ describe('S3 Flow tests', () => { }); }); - test('Source and Destination with a bucket name', () => { + test('Destination with complex formatting ', () => { const stack = new Stack(undefined, undefined); const bucket = Bucket.fromBucketName(stack, 'TestBucket', 'test-bucket'); @@ -143,9 +145,7 @@ describe('S3 Flow tests', () => { }, FileType: 'CSV', PrefixConfig: { - PathPrefixHierarchy: [ - 'SCHEMA_VERSION', - ], + PathPrefixHierarchy: ['SCHEMA_VERSION'], PrefixFormat: 'HOUR', }, }, @@ -172,7 +172,311 @@ describe('S3 Flow tests', () => { TaskProperties: [ { Key: 'EXCLUDE_SOURCE_FIELDS_LIST', - Value: '[\"field1\",\"field2\"]', + Value: '["field1","field2"]', + }, + ], + TaskType: 'Map_all', + }, + ], + TriggerConfig: { + TriggerType: 'OnDemand', + }, + }); + }); + + test('Destination with catalog', () => { + const stack = new Stack(undefined, undefined); + + const bucket = Bucket.fromBucketName(stack, 'TestBucket', 'test-bucket'); + + const source = new S3Source({ + bucket: bucket, + prefix: 'source-prefix', + format: { + type: S3InputFileType.JSON, + }, + }); + + const destination = new S3Destination({ + location: { bucket: bucket }, + catalog: { + database: new Database(stack, 'TestDb', { + databaseName: 'testdb', + }), + tablePrefix: 'tablePrefix', + }, + }); + + new OnDemandFlow(stack, 'testFlow', { + source: source, + destination, + mappings: [Mapping.mapAll({ exclude: ['field1', 'field2'] })], + }); + + const template = Template.fromStack(stack); + + template.resourceCountIs('AWS::S3::Bucket', 0); + + template.resourceCountIs('AWS::AppFlow::Flow', 1); + template.hasResourceProperties('AWS::AppFlow::Flow', { + DestinationFlowConfigList: [ + { + ConnectorType: 'S3', + DestinationConnectorProperties: { + S3: { + BucketName: 'test-bucket', + }, + }, + }, + ], + MetadataCatalogConfig: { + GlueDataCatalog: { + DatabaseName: { + Ref: 'TestDb6EF03243', + }, + RoleArn: { + 'Fn::GetAtt': ['testFlowGlueAccessRole34DC638A', 'Arn'], + }, + TablePrefix: 'tablePrefix', + }, + }, + FlowName: 'testFlow', + SourceFlowConfig: { + ConnectorType: 'S3', + SourceConnectorProperties: { + S3: { + BucketName: 'test-bucket', + BucketPrefix: 'source-prefix', + }, + }, + }, + Tasks: [ + { + ConnectorOperator: { + S3: 'NO_OP', + }, + SourceFields: [], + TaskProperties: [ + { + Key: 'EXCLUDE_SOURCE_FIELDS_LIST', + Value: '["field1","field2"]', + }, + ], + TaskType: 'Map_all', + }, + ], + TriggerConfig: { + TriggerType: 'OnDemand', + }, + }); + + template.resourceCountIs('AWS::IAM::Role', 1); + template.hasResourceProperties('AWS::IAM::Role', { + AssumeRolePolicyDocument: { + Statement: [ + { + Action: 'sts:AssumeRole', + Effect: 'Allow', + Principal: { + Service: 'appflow.amazonaws.com', + }, + }, + ], + }, + }); + + template.resourceCountIs('AWS::IAM::Policy', 1); + template.hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: Match.arrayEquals([ + 'glue:BatchCreatePartition', + 'glue:CreatePartitionIndex', + 'glue:DeleteDatabase', + 'glue:GetTableVersions', + 'glue:GetPartitions', + 'glue:BatchDeletePartition', + 'glue:DeleteTableVersion', + 'glue:UpdateTable', + 'glue:DeleteTable', + 'glue:DeletePartitionIndex', + 'glue:GetTableVersion', + 'glue:CreatePartition', + 'glue:UntagResource', + 'glue:UpdatePartition', + 'glue:TagResource', + 'glue:UpdateDatabase', + 'glue:CreateTable', + 'glue:BatchUpdatePartition', + 'glue:GetTables', + 'glue:BatchGetPartition', + 'glue:GetDatabases', + 'glue:GetPartitionIndexes', + 'glue:GetTable', + 'glue:GetDatabase', + 'glue:GetPartition', + 'glue:CreateDatabase', + 'glue:BatchDeleteTableVersion', + 'glue:BatchDeleteTable', + 'glue:DeletePartition', + ]), + Effect: 'Allow', + Resource: Match.arrayEquals([ + { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { Ref: 'AWS::Region' }, + ':', + { Ref: 'AWS::AccountId' }, + ':catalog', + ], + ], + }, + { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { Ref: 'AWS::Region' }, + ':', + { Ref: 'AWS::AccountId' }, + ':database/', + { Ref: 'TestDb6EF03243' }, + ], + ], + }, + { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { Ref: 'AWS::Region' }, + ':', + { Ref: 'AWS::AccountId' }, + ':table/', + { Ref: 'TestDb6EF03243' }, + '/tablePrefix*', + ], + ], + }, + ]), + }, + ], + }, + }); + + template.resourceCountIs('AWS::Glue::Database', 1); + }); + + test('Destination with catalog with custom role and imported values', () => { + const stack = new Stack(undefined, undefined); + + const bucket = Bucket.fromBucketName(stack, 'TestBucket', 'test-bucket'); + const database = Database.fromDatabaseArn( + stack, + 'TestDb', + stack.formatArn({ + service: 'glue', + resource: 'database', + resourceName: 'test-db', + }), + ); + const role = Role.fromRoleName(stack, 'TestRole', 'test-role'); + + const source = new S3Source({ + bucket: bucket, + prefix: 'source-prefix', + format: { + type: S3InputFileType.JSON, + }, + }); + + const destination = new S3Destination({ + location: { bucket: bucket }, + catalog: { + role, + database, + tablePrefix: 'tablePrefix', + }, + }); + + new OnDemandFlow(stack, 'testFlow', { + source: source, + destination, + mappings: [Mapping.mapAll({ exclude: ['field1', 'field2'] })], + }); + + const template = Template.fromStack(stack); + + template.resourceCountIs('AWS::S3::Bucket', 0); + template.resourceCountIs('AWS::AppFlow::Flow', 1); + template.resourceCountIs('AWS::IAM::Role', 0); + template.resourceCountIs('AWS::Glue::Database', 0); + + template.hasResourceProperties('AWS::AppFlow::Flow', { + DestinationFlowConfigList: [ + { + ConnectorType: 'S3', + DestinationConnectorProperties: { + S3: { + BucketName: 'test-bucket', + }, + }, + }, + ], + MetadataCatalogConfig: { + GlueDataCatalog: { + DatabaseName: 'test-db', + RoleArn: { + 'Fn::Join': [ + '', + [ + 'arn:', + { Ref: 'AWS::Partition' }, + ':iam::', + { Ref: 'AWS::AccountId' }, + ':role/test-role', + ], + ], + }, + TablePrefix: 'tablePrefix', + }, + }, + FlowName: 'testFlow', + SourceFlowConfig: { + ConnectorType: 'S3', + SourceConnectorProperties: { + S3: { + BucketName: 'test-bucket', + BucketPrefix: 'source-prefix', + }, + }, + }, + Tasks: [ + { + ConnectorOperator: { + S3: 'NO_OP', + }, + SourceFields: [], + TaskProperties: [ + { + Key: 'EXCLUDE_SOURCE_FIELDS_LIST', + Value: '["field1","field2"]', }, ], TaskType: 'Map_all', @@ -183,4 +487,4 @@ describe('S3 Flow tests', () => { }, }); }); -}); \ No newline at end of file +});