From bb50b0df184bd2fc61d701af152f6ec23ee0de43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Paw=C5=82aszek?= Date: Thu, 26 Oct 2023 23:55:25 +0200 Subject: [PATCH] chore: correcting the ConnectorLabel + refac + tests --- .gitignore | 8 + .npmignore | 1 + .projen/tasks.json | 69 +++ package.json | 5 + src/microsoftsharepointonline/profile.ts | 7 +- src/microsoftsharepointonline/source.ts | 18 +- src/microsoftsharepointonline/type.ts | 2 +- src/microsoftsharepointonline/util.ts | 4 +- .../TestStack.assets.json | 32 ++ .../TestStack.template.json | 509 ++++++++++++++++++ ...d-microsoftsharepointonline-to-s3.integ.ts | 61 +++ .../microsoftsharepointonline/profile.test.ts | 6 +- test/microsoftsharepointonline/source.test.ts | 27 +- 13 files changed, 733 insertions(+), 16 deletions(-) create mode 100644 test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.assets.json create mode 100644 test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.template.json create mode 100644 test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts diff --git a/.gitignore b/.gitignore index cf533e6a..be9f1640 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,14 @@ test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/manifest.json test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/**/manifest.json test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/tree.json test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/**/tree.json +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/asset.* +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/**/asset.* +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/cdk.out +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/**/cdk.out +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/manifest.json +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/**/manifest.json +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/tree.json +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/**/tree.json test/integ/ondemand-s3-to-snowflake.integ.snapshot/asset.* test/integ/ondemand-s3-to-snowflake.integ.snapshot/**/asset.* test/integ/ondemand-s3-to-snowflake.integ.snapshot/cdk.out diff --git a/.npmignore b/.npmignore index 8fbe9d00..7d61db08 100644 --- a/.npmignore +++ b/.npmignore @@ -24,6 +24,7 @@ tsconfig.tsbuildinfo !.jsii !/assets/ test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot test/integ/ondemand-s3-to-snowflake.integ.snapshot test/integ/ondemand-salesforce-to-redshift.integ.snapshot test/integ/ondemand-salesforce-to-s3.integ.snapshot diff --git a/.projen/tasks.json b/.projen/tasks.json index f98998fe..17132b6c 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -248,6 +248,69 @@ } ] }, + "integ:ondemand-microsoftsharepointonline-to-s3:assert": { + "name": "integ:ondemand-microsoftsharepointonline-to-s3:assert", + "description": "assert the snapshot of integration test 'ondemand-microsoftsharepointonline-to-s3'", + "steps": [ + { + "exec": "[ -d \"test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot\" ] || (echo \"No snapshot available for integration test 'ondemand-microsoftsharepointonline-to-s3'. Run 'projen integ:ondemand-microsoftsharepointonline-to-s3:deploy' to capture.\" && exit 1)" + }, + { + "exec": "cdk synth --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata -o test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/assert.cdk.out > /dev/null" + }, + { + "exec": "diff -r -x asset.* -x cdk.out -x manifest.json -x tree.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/ test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/assert.cdk.out/" + } + ] + }, + "integ:ondemand-microsoftsharepointonline-to-s3:deploy": { + "name": "integ:ondemand-microsoftsharepointonline-to-s3:deploy", + "description": "deploy integration test 'ondemand-microsoftsharepointonline-to-s3' and capture snapshot", + "steps": [ + { + "exec": "rm -fr test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/deploy.cdk.out" + }, + { + "exec": "cdk deploy --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata '**' --require-approval=never -o test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/deploy.cdk.out" + }, + { + "exec": "rm -fr test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot" + }, + { + "exec": "mv test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/deploy.cdk.out test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot" + }, + { + "spawn": "integ:ondemand-microsoftsharepointonline-to-s3:destroy" + } + ] + }, + "integ:ondemand-microsoftsharepointonline-to-s3:destroy": { + "name": "integ:ondemand-microsoftsharepointonline-to-s3:destroy", + "description": "destroy integration test 'ondemand-microsoftsharepointonline-to-s3'", + "steps": [ + { + "exec": "cdk destroy --app test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot '**' --no-version-reporting" + } + ] + }, + "integ:ondemand-microsoftsharepointonline-to-s3:snapshot": { + "name": "integ:ondemand-microsoftsharepointonline-to-s3:snapshot", + "description": "update snapshot for integration test \"ondemand-microsoftsharepointonline-to-s3\"", + "steps": [ + { + "exec": "cdk synth --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata -o test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot > /dev/null" + } + ] + }, + "integ:ondemand-microsoftsharepointonline-to-s3:watch": { + "name": "integ:ondemand-microsoftsharepointonline-to-s3:watch", + "description": "watch integration test 'ondemand-microsoftsharepointonline-to-s3' (without updating snapshots)", + "steps": [ + { + "exec": "cdk watch --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata '**' -o test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/deploy.cdk.out" + } + ] + }, "integ:ondemand-s3-to-snowflake:assert": { "name": "integ:ondemand-s3-to-snowflake:assert", "description": "assert the snapshot of integration test 'ondemand-s3-to-snowflake'", @@ -759,6 +822,9 @@ { "spawn": "integ:ondemand-googleanalytics4-to-s3:snapshot" }, + { + "spawn": "integ:ondemand-microsoftsharepointonline-to-s3:snapshot" + }, { "spawn": "integ:ondemand-s3-to-snowflake:snapshot" }, @@ -942,6 +1008,9 @@ { "spawn": "integ:ondemand-googleanalytics4-to-s3:assert" }, + { + "spawn": "integ:ondemand-microsoftsharepointonline-to-s3:assert" + }, { "spawn": "integ:ondemand-s3-to-snowflake:assert" }, diff --git a/package.json b/package.json index 55c93030..7b5f564b 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,11 @@ "integ:ondemand-googleanalytics4-to-s3:destroy": "npx projen integ:ondemand-googleanalytics4-to-s3:destroy", "integ:ondemand-googleanalytics4-to-s3:snapshot": "npx projen integ:ondemand-googleanalytics4-to-s3:snapshot", "integ:ondemand-googleanalytics4-to-s3:watch": "npx projen integ:ondemand-googleanalytics4-to-s3:watch", + "integ:ondemand-microsoftsharepointonline-to-s3:assert": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:assert", + "integ:ondemand-microsoftsharepointonline-to-s3:deploy": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:deploy", + "integ:ondemand-microsoftsharepointonline-to-s3:destroy": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:destroy", + "integ:ondemand-microsoftsharepointonline-to-s3:snapshot": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:snapshot", + "integ:ondemand-microsoftsharepointonline-to-s3:watch": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:watch", "integ:ondemand-s3-to-snowflake:assert": "npx projen integ:ondemand-s3-to-snowflake:assert", "integ:ondemand-s3-to-snowflake:deploy": "npx projen integ:ondemand-s3-to-snowflake:deploy", "integ:ondemand-s3-to-snowflake:destroy": "npx projen integ:ondemand-s3-to-snowflake:destroy", diff --git a/src/microsoftsharepointonline/profile.ts b/src/microsoftsharepointonline/profile.ts index 235a963c..2eb624d9 100644 --- a/src/microsoftsharepointonline/profile.ts +++ b/src/microsoftsharepointonline/profile.ts @@ -5,6 +5,7 @@ SPDX-License-Identifier: Apache-2.0 import { CfnConnectorProfile } from 'aws-cdk-lib/aws-appflow'; import { Construct } from 'constructs'; import { MicrosoftSharepointOnlineConnectorType } from './type'; +import { MicrosoftSharepointOnlineTokenUrlBuilder } from './util'; import { ConnectorAuthenticationType } from '../core/connectors/connector-authentication-type'; import { ConnectorProfileBase, ConnectorProfileProps } from '../core/connectors/connector-profile'; import { OAuth2GrantType as OAuthGrantType } from '../core/connectors/oauth2-granttype'; @@ -38,7 +39,7 @@ export interface MicrosoftSharepointOnlineOAuthSettings { readonly flow?: MicrosoftSharepointOnlineOAuthFlow; - readonly endpoints: MicrosoftSharepointOnlineOAuthEndpointsSettings; + readonly endpoints?: MicrosoftSharepointOnlineOAuthEndpointsSettings; } export class MicrosoftSharepointOnlineConnectorProfile extends ConnectorProfileBase { @@ -51,6 +52,8 @@ export class MicrosoftSharepointOnlineConnectorProfile extends ConnectorProfileB return this._fromConnectorProfileAttributes(scope, id, { name }) as MicrosoftSharepointOnlineConnectorProfile; } + private static readonly defaultTokenEndpoint = MicrosoftSharepointOnlineTokenUrlBuilder.build(); + constructor(scope: Construct, id: string, props: MicrosoftSharepointOnlineConnectorProfileProps) { super(scope, id, props, MicrosoftSharepointOnlineConnectorType.instance); } @@ -66,7 +69,7 @@ export class MicrosoftSharepointOnlineConnectorProfile extends ConnectorProfileB oAuth2GrantType: OAuthGrantType.AUTHORIZATION_CODE, // INFO: even if we provide only the access token this property is required // TODO: think about if this is correct. token can be IResolvable - tokenUrl: properties.oAuth.endpoints?.token, + tokenUrl: properties.oAuth.endpoints?.token ?? MicrosoftSharepointOnlineConnectorProfile.defaultTokenEndpoint, }, }, }; diff --git a/src/microsoftsharepointonline/source.ts b/src/microsoftsharepointonline/source.ts index 7d9d3ffa..279f1394 100644 --- a/src/microsoftsharepointonline/source.ts +++ b/src/microsoftsharepointonline/source.ts @@ -2,6 +2,7 @@ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 */ +import { Fn } from 'aws-cdk-lib'; import { CfnFlow } from 'aws-cdk-lib/aws-appflow'; import { IConstruct } from 'constructs'; import { MicrosoftSharepointOnlineConnectorProfile } from './profile'; @@ -10,13 +11,18 @@ import { ConnectorType } from '../core/connectors/connector-type'; import { IFlow } from '../core/flows'; import { ISource } from '../core/vertices'; +export interface MicrosoftSharepointObject { + readonly site: string; + readonly drives: string[]; +} + /** * Properties of a Google Analytics v4 Source */ export interface MicrosoftSharepointOnlineSourceProps { readonly profile: MicrosoftSharepointOnlineConnectorProfile; readonly apiVersion: string; - readonly object: string; + readonly object: MicrosoftSharepointObject; } /** @@ -44,9 +50,17 @@ export class MicrosoftSharepointOnlineSource implements ISource { } private buildSourceConnectorProperties(): CfnFlow.SourceConnectorPropertiesProperty { + + if (this.props.object.drives.length < 1) { + throw new Error('At least one drive must be specified'); + } + return { customConnector: { - entityName: this.props.object, + entityName: this.props.object.site, + customProperties: { + subEntities: `["${Fn.join('","', this.props.object.drives)}"]`, + }, }, }; } diff --git a/src/microsoftsharepointonline/type.ts b/src/microsoftsharepointonline/type.ts index c5dc791a..b3f66aca 100644 --- a/src/microsoftsharepointonline/type.ts +++ b/src/microsoftsharepointonline/type.ts @@ -19,6 +19,6 @@ export class MicrosoftSharepointOnlineConnectorType extends ConnectorType { private static actualInstance: ConnectorType; constructor() { - super('MS_SharePoint_Connector', true); + super('MicrosoftSharePointOnline', true); } } \ No newline at end of file diff --git a/src/microsoftsharepointonline/util.ts b/src/microsoftsharepointonline/util.ts index c56c3094..b2909641 100644 --- a/src/microsoftsharepointonline/util.ts +++ b/src/microsoftsharepointonline/util.ts @@ -3,8 +3,8 @@ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 */ export class MicrosoftSharepointOnlineTokenUrlBuilder { - public static buildFromTenant(tenantId: string) { - return `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`; + public static build(tenantId?: string) { + return `https://login.microsoftonline.com/${tenantId ?? 'common'}/oauth2/v2.0/token`; } } diff --git a/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.assets.json new file mode 100644 index 00000000..eff0f0b3 --- /dev/null +++ b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.assets.json @@ -0,0 +1,32 @@ +{ + "version": "32.0.0", + "files": { + "350185a1069fa20a23a583e20c77f6844218bd73097902362dc94f1a108f5d89": { + "source": { + "path": "asset.350185a1069fa20a23a583e20c77f6844218bd73097902362dc94f1a108f5d89", + "packaging": "zip" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "350185a1069fa20a23a583e20c77f6844218bd73097902362dc94f1a108f5d89.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "6b57b7d92e5a078619ebade6f916f2f97a85517dd96d351c12af4d3b69ccaa6c": { + "source": { + "path": "TestStack.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "6b57b7d92e5a078619ebade6f916f2f97a85517dd96d351c12af4d3b69ccaa6c.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.template.json b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.template.json new file mode 100644 index 00000000..6e9cc8ba --- /dev/null +++ b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.template.json @@ -0,0 +1,509 @@ +{ + "Resources": { + "TestConnectorEE9F0A13": { + "Type": "AWS::AppFlow::ConnectorProfile", + "Properties": { + "ConnectionMode": "Public", + "ConnectorProfileName": "TestConnector", + "ConnectorType": "CustomConnector", + "ConnectorLabel": "MicrosoftSharePointOnline", + "ConnectorProfileConfig": { + "ConnectorProfileCredentials": { + "CustomConnector": { + "AuthenticationType": "OAUTH2", + "Oauth2": { + "AccessToken": { + "Fn::Join": [ + "", + [ + "{{resolve:secretsmanager:arn:", + { + "Ref": "AWS::Partition" + }, + ":secretsmanager:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":secret:appflow/mssharepointonline:SecretString:accessToken::}}" + ] + ] + }, + "RefreshToken": { + "Fn::Join": [ + "", + [ + "{{resolve:secretsmanager:arn:", + { + "Ref": "AWS::Partition" + }, + ":secretsmanager:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":secret:appflow/mssharepointonline:SecretString:refreshToken::}}" + ] + ] + } + } + } + }, + "ConnectorProfileProperties": { + "CustomConnector": { + "OAuth2Properties": { + "OAuth2GrantType": "AUTHORIZATION_CODE", + "TokenUrl": { + "Fn::Join": [ + "", + [ + "https://login.microsoftonline.com/{{resolve:secretsmanager:arn:", + { + "Ref": "AWS::Partition" + }, + ":secretsmanager:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":secret:appflow/mssharepointonline:SecretString:tenantId::}}/oauth2/v2.0/token" + ] + ] + } + } + } + } + } + } + }, + "TestBucket560B80BC": { + "Type": "AWS::S3::Bucket", + "Properties": { + "Tags": [ + { + "Key": "aws-cdk:auto-delete-objects", + "Value": "true" + } + ] + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "TestBucketPolicyBA12ED38": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "TestBucket560B80BC" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "TestBucket560B80BC", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "TestBucket560B80BC", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:PutObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:ListBucketMultipartUploads", + "s3:GetBucketAcl", + "s3:PutObjectAcl" + ], + "Condition": { + "StringEquals": { + "aws:SourceAccount": { + "Ref": "AWS::AccountId" + } + } + }, + "Effect": "Allow", + "Principal": { + "Service": "appflow.amazonaws.com" + }, + "Resource": [ + { + "Fn::GetAtt": [ + "TestBucket560B80BC", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "TestBucket560B80BC", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "TestBucketAutoDeleteObjectsCustomResource8FEAABD5": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "TestBucket560B80BC" + } + }, + "DependsOn": [ + "TestBucketPolicyBA12ED38" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ] + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + } + ] + } + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "350185a1069fa20a23a583e20c77f6844218bd73097902362dc94f1a108f5d89.zip" + }, + "Timeout": 900, + "MemorySize": 128, + "Handler": "__entrypoint__.handler", + "Role": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + }, + "Runtime": "nodejs16.x", + "Description": { + "Fn::Join": [ + "", + [ + "Lambda function for auto-deleting objects in ", + { + "Ref": "TestBucket560B80BC" + }, + " S3 bucket." + ] + ] + } + }, + "DependsOn": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" + ] + }, + "OnDemandFlow4ECA54C5": { + "Type": "AWS::AppFlow::Flow", + "Properties": { + "DestinationFlowConfigList": [ + { + "ConnectorType": "S3", + "DestinationConnectorProperties": { + "S3": { + "BucketName": { + "Ref": "TestBucket560B80BC" + }, + "S3OutputFormatConfig": { + "AggregationConfig": { + "AggregationType": "SingleFile" + }, + "FileType": "JSON" + } + } + } + } + ], + "FlowName": "OnDemandFlow", + "SourceFlowConfig": { + "ApiVersion": "v1.0", + "ConnectorProfileName": "TestConnector", + "ConnectorType": "CustomConnector", + "SourceConnectorProperties": { + "CustomConnector": { + "CustomProperties": { + "subEntities": { + "Fn::Join": [ + "", + [ + "[\"{{resolve:secretsmanager:arn:", + { + "Ref": "AWS::Partition" + }, + ":secretsmanager:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":secret:appflow/mssharepointonline:SecretString:drive::}}\"]" + ] + ] + } + }, + "EntityName": { + "Fn::Join": [ + "", + [ + "{{resolve:secretsmanager:arn:", + { + "Ref": "AWS::Partition" + }, + ":secretsmanager:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":secret:appflow/mssharepointonline:SecretString:site::}}" + ] + ] + } + } + } + }, + "Tasks": [ + { + "ConnectorOperator": { + "CustomConnector": "NO_OP" + }, + "SourceFields": [], + "TaskProperties": [ + { + "Key": "EXCLUDE_SOURCE_FIELDS_LIST", + "Value": "[]" + } + ], + "TaskType": "Map_all" + } + ], + "TriggerConfig": { + "TriggerType": "OnDemand" + } + }, + "DependsOn": [ + "TestBucketAutoDeleteObjectsCustomResource8FEAABD5", + "TestBucketPolicyBA12ED38", + "TestBucket560B80BC", + "TestConnectorEE9F0A13" + ] + } + }, + "Mappings": { + "DefaultCrNodeVersionMap": { + "af-south-1": { + "value": "nodejs16.x" + }, + "ap-east-1": { + "value": "nodejs16.x" + }, + "ap-northeast-1": { + "value": "nodejs16.x" + }, + "ap-northeast-2": { + "value": "nodejs16.x" + }, + "ap-northeast-3": { + "value": "nodejs16.x" + }, + "ap-south-1": { + "value": "nodejs16.x" + }, + "ap-south-2": { + "value": "nodejs16.x" + }, + "ap-southeast-1": { + "value": "nodejs16.x" + }, + "ap-southeast-2": { + "value": "nodejs16.x" + }, + "ap-southeast-3": { + "value": "nodejs16.x" + }, + "ca-central-1": { + "value": "nodejs16.x" + }, + "cn-north-1": { + "value": "nodejs16.x" + }, + "cn-northwest-1": { + "value": "nodejs16.x" + }, + "eu-central-1": { + "value": "nodejs16.x" + }, + "eu-central-2": { + "value": "nodejs16.x" + }, + "eu-north-1": { + "value": "nodejs16.x" + }, + "eu-south-1": { + "value": "nodejs16.x" + }, + "eu-south-2": { + "value": "nodejs16.x" + }, + "eu-west-1": { + "value": "nodejs16.x" + }, + "eu-west-2": { + "value": "nodejs16.x" + }, + "eu-west-3": { + "value": "nodejs16.x" + }, + "me-central-1": { + "value": "nodejs16.x" + }, + "me-south-1": { + "value": "nodejs16.x" + }, + "sa-east-1": { + "value": "nodejs16.x" + }, + "us-east-1": { + "value": "nodejs16.x" + }, + "us-east-2": { + "value": "nodejs16.x" + }, + "us-gov-east-1": { + "value": "nodejs16.x" + }, + "us-gov-west-1": { + "value": "nodejs16.x" + }, + "us-iso-east-1": { + "value": "nodejs14.x" + }, + "us-iso-west-1": { + "value": "nodejs14.x" + }, + "us-isob-east-1": { + "value": "nodejs14.x" + }, + "us-west-1": { + "value": "nodejs16.x" + }, + "us-west-2": { + "value": "nodejs16.x" + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts new file mode 100644 index 00000000..0de4106f --- /dev/null +++ b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts @@ -0,0 +1,61 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { App, RemovalPolicy, Stack } from 'aws-cdk-lib'; +import { Bucket } from 'aws-cdk-lib/aws-s3'; +import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; +import { MicrosoftSharepointOnlineApiVersion, MicrosoftSharepointOnlineConnectorProfile, MicrosoftSharepointOnlineSource, Mapping, OnDemandFlow, S3Destination, MicrosoftSharepointOnlineTokenUrlBuilder, S3OutputAggregationType } from '../../src'; + +const app = new App({ + treeMetadata: false, +}); + +const stack = new Stack(app, 'TestStack'); + +const secret = Secret.fromSecretNameV2(stack, 'TestSecret', 'appflow/mssharepointonline'); + +const profile = new MicrosoftSharepointOnlineConnectorProfile(stack, 'TestConnector', { + oAuth: { + accessToken: secret.secretValueFromJson('accessToken').toString(), + flow: { + refreshTokenGrant: { + refreshToken: secret.secretValueFromJson('refreshToken').toString(), + }, + }, + endpoints: { + token: MicrosoftSharepointOnlineTokenUrlBuilder.build(secret.secretValueFromJson('tenantId').toString()), + }, + }, +}); + +const source = new MicrosoftSharepointOnlineSource({ + profile: profile, + apiVersion: MicrosoftSharepointOnlineApiVersion.V1, + object: { + site: secret.secretValueFromJson('site').toString(), + drives: [secret.secretValueFromJson('drive').toString()], + }, +}); + +const bucket = new Bucket(stack, 'TestBucket', { + autoDeleteObjects: true, + removalPolicy: RemovalPolicy.DESTROY, +}); + +const destination = new S3Destination({ + location: { bucket }, + formatting: { + aggregation: { + type: S3OutputAggregationType.SINGLE_FILE, + }, + }, +}); + +new OnDemandFlow(stack, 'OnDemandFlow', { + source: source, + destination: destination, + mappings: [Mapping.mapAll()], +}); + +app.synth(); diff --git a/test/microsoftsharepointonline/profile.test.ts b/test/microsoftsharepointonline/profile.test.ts index 26b55b74..479bdff6 100644 --- a/test/microsoftsharepointonline/profile.test.ts +++ b/test/microsoftsharepointonline/profile.test.ts @@ -36,7 +36,7 @@ describe('MicrosoftSharepointOnlineConnectorProfile', () => { ConnectionMode: 'Public', ConnectorProfileName: 'TestProfile', ConnectorType: 'CustomConnector', - ConnectorLabel: 'MS_SharePoint_Connector', + ConnectorLabel: 'MicrosoftSharePointOnline', ConnectorProfileConfig: { ConnectorProfileCredentials: { CustomConnector: { @@ -86,7 +86,7 @@ describe('MicrosoftSharepointOnlineConnectorProfile', () => { ConnectionMode: 'Public', ConnectorProfileName: 'TestProfile', ConnectorType: 'CustomConnector', - ConnectorLabel: 'MS_SharePoint_Connector', + ConnectorLabel: 'MicrosoftSharePointOnline', ConnectorProfileConfig: { ConnectorProfileCredentials: { CustomConnector: { @@ -196,7 +196,7 @@ describe('MicrosoftSharepointOnlineConnectorProfile', () => { ConnectionMode: 'Public', ConnectorProfileName: 'TestProfile', ConnectorType: 'CustomConnector', - ConnectorLabel: 'MS_SharePoint_Connector', + ConnectorLabel: 'MicrosoftSharePointOnline', ConnectorProfileConfig: { ConnectorProfileCredentials: { CustomConnector: { diff --git a/test/microsoftsharepointonline/source.test.ts b/test/microsoftsharepointonline/source.test.ts index 14201b51..b443fde2 100644 --- a/test/microsoftsharepointonline/source.test.ts +++ b/test/microsoftsharepointonline/source.test.ts @@ -22,7 +22,10 @@ describe('MicrosoftSharepointOnlineSource', () => { const stack = new Stack(undefined, 'TestStack'); const source = new MicrosoftSharepointOnlineSource({ profile: MicrosoftSharepointOnlineConnectorProfile.fromConnectionProfileName(stack, 'TestProfile', 'dummy-profile'), - object: 'dummy-object', + object: { + site: 'sites/dummysite.sharepoint.com,3f42g340-bc23-4a31-b7e5-722e57c39cb8,5bbc39fb-2b17-423b-a007-40ca508389a5', + drives: ['drives/b!fcPDltwTLSougEJuDFjE?U5qHuXbkzlvSaA5oNoMW4tB0y6mebcx9m-ckwA9KtKE'], + }, apiVersion: MicrosoftSharepointOnlineApiVersion.V1, }); @@ -37,7 +40,10 @@ describe('MicrosoftSharepointOnlineSource', () => { const stack = new Stack(undefined, 'TestStack'); const source = new MicrosoftSharepointOnlineSource({ profile: MicrosoftSharepointOnlineConnectorProfile.fromConnectionProfileName(stack, 'TestProfile', 'dummy-profile'), - object: 'dummy-object', + object: { + site: 'sites/dummysite.sharepoint.com,3f42g340-bc23-4a31-b7e5-722e57c39cb8,5bbc39fb-2b17-423b-a007-40ca508389a5', + drives: ['drives/b!fcPDltwTLSougEJuDFjE?U5qHuXbkzlvSaA5oNoMW4tB0y6mebcx9m-ckwA9KtKE'], + }, apiVersion: MicrosoftSharepointOnlineApiVersion.V1, }); @@ -71,7 +77,10 @@ describe('MicrosoftSharepointOnlineSource', () => { ConnectorType: 'CustomConnector', SourceConnectorProperties: { CustomConnector: { - EntityName: 'dummy-object', + EntityName: 'sites/dummysite.sharepoint.com,3f42g340-bc23-4a31-b7e5-722e57c39cb8,5bbc39fb-2b17-423b-a007-40ca508389a5', + CustomProperties: { + subEntities: '["drives/b!fcPDltwTLSougEJuDFjE?U5qHuXbkzlvSaA5oNoMW4tB0y6mebcx9m-ckwA9KtKE"]', + }, }, }, }, @@ -117,7 +126,10 @@ describe('MicrosoftSharepointOnlineSource', () => { const source = new MicrosoftSharepointOnlineSource({ profile: profile, - object: 'dummy-object', + object: { + site: 'sites/dummysite.sharepoint.com,3f42g340-bc23-4a31-b7e5-722e57c39cb8,5bbc39fb-2b17-423b-a007-40ca508389a5', + drives: ['drives/b!fcPDltwTLSougEJuDFjE?U5qHuXbkzlvSaA5oNoMW4tB0y6mebcx9m-ckwA9KtKE'], + }, apiVersion: MicrosoftSharepointOnlineApiVersion.V1, }); @@ -137,7 +149,7 @@ describe('MicrosoftSharepointOnlineSource', () => { ConnectionMode: 'Public', ConnectorProfileName: 'TestProfile', ConnectorType: 'CustomConnector', - ConnectorLabel: 'MS_SharePoint_Connector', + ConnectorLabel: 'MicrosoftSharePointOnline', ConnectorProfileConfig: { ConnectorProfileCredentials: { CustomConnector: { @@ -182,7 +194,10 @@ describe('MicrosoftSharepointOnlineSource', () => { ConnectorType: 'CustomConnector', SourceConnectorProperties: { CustomConnector: { - EntityName: 'dummy-object', + EntityName: 'sites/dummysite.sharepoint.com,3f42g340-bc23-4a31-b7e5-722e57c39cb8,5bbc39fb-2b17-423b-a007-40ca508389a5', + CustomProperties: { + subEntities: '["drives/b!fcPDltwTLSougEJuDFjE?U5qHuXbkzlvSaA5oNoMW4tB0y6mebcx9m-ckwA9KtKE"]', + }, }, }, },