diff --git a/.gitignore b/.gitignore index cf533e6a..be9f1640 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,14 @@ test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/manifest.json test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/**/manifest.json test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/tree.json test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/**/tree.json +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/asset.* +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/**/asset.* +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/cdk.out +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/**/cdk.out +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/manifest.json +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/**/manifest.json +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/tree.json +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/**/tree.json test/integ/ondemand-s3-to-snowflake.integ.snapshot/asset.* test/integ/ondemand-s3-to-snowflake.integ.snapshot/**/asset.* test/integ/ondemand-s3-to-snowflake.integ.snapshot/cdk.out diff --git a/.npmignore b/.npmignore index 8fbe9d00..7d61db08 100644 --- a/.npmignore +++ b/.npmignore @@ -24,6 +24,7 @@ tsconfig.tsbuildinfo !.jsii !/assets/ test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot +test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot test/integ/ondemand-s3-to-snowflake.integ.snapshot test/integ/ondemand-salesforce-to-redshift.integ.snapshot test/integ/ondemand-salesforce-to-s3.integ.snapshot diff --git a/.projen/tasks.json b/.projen/tasks.json index f98998fe..17132b6c 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -248,6 +248,69 @@ } ] }, + "integ:ondemand-microsoftsharepointonline-to-s3:assert": { + "name": "integ:ondemand-microsoftsharepointonline-to-s3:assert", + "description": "assert the snapshot of integration test 'ondemand-microsoftsharepointonline-to-s3'", + "steps": [ + { + "exec": "[ -d \"test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot\" ] || (echo \"No snapshot available for integration test 'ondemand-microsoftsharepointonline-to-s3'. Run 'projen integ:ondemand-microsoftsharepointonline-to-s3:deploy' to capture.\" && exit 1)" + }, + { + "exec": "cdk synth --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata -o test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/assert.cdk.out > /dev/null" + }, + { + "exec": "diff -r -x asset.* -x cdk.out -x manifest.json -x tree.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/ test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/assert.cdk.out/" + } + ] + }, + "integ:ondemand-microsoftsharepointonline-to-s3:deploy": { + "name": "integ:ondemand-microsoftsharepointonline-to-s3:deploy", + "description": "deploy integration test 'ondemand-microsoftsharepointonline-to-s3' and capture snapshot", + "steps": [ + { + "exec": "rm -fr test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/deploy.cdk.out" + }, + { + "exec": "cdk deploy --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata '**' --require-approval=never -o test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/deploy.cdk.out" + }, + { + "exec": "rm -fr test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot" + }, + { + "exec": "mv test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/deploy.cdk.out test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot" + }, + { + "spawn": "integ:ondemand-microsoftsharepointonline-to-s3:destroy" + } + ] + }, + "integ:ondemand-microsoftsharepointonline-to-s3:destroy": { + "name": "integ:ondemand-microsoftsharepointonline-to-s3:destroy", + "description": "destroy integration test 'ondemand-microsoftsharepointonline-to-s3'", + "steps": [ + { + "exec": "cdk destroy --app test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot '**' --no-version-reporting" + } + ] + }, + "integ:ondemand-microsoftsharepointonline-to-s3:snapshot": { + "name": "integ:ondemand-microsoftsharepointonline-to-s3:snapshot", + "description": "update snapshot for integration test \"ondemand-microsoftsharepointonline-to-s3\"", + "steps": [ + { + "exec": "cdk synth --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata -o test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot > /dev/null" + } + ] + }, + "integ:ondemand-microsoftsharepointonline-to-s3:watch": { + "name": "integ:ondemand-microsoftsharepointonline-to-s3:watch", + "description": "watch integration test 'ondemand-microsoftsharepointonline-to-s3' (without updating snapshots)", + "steps": [ + { + "exec": "cdk watch --app \"ts-node -P tsconfig.dev.json test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata '**' -o test/integ/.tmp/ondemand-microsoftsharepointonline-to-s3.integ/deploy.cdk.out" + } + ] + }, "integ:ondemand-s3-to-snowflake:assert": { "name": "integ:ondemand-s3-to-snowflake:assert", "description": "assert the snapshot of integration test 'ondemand-s3-to-snowflake'", @@ -759,6 +822,9 @@ { "spawn": "integ:ondemand-googleanalytics4-to-s3:snapshot" }, + { + "spawn": "integ:ondemand-microsoftsharepointonline-to-s3:snapshot" + }, { "spawn": "integ:ondemand-s3-to-snowflake:snapshot" }, @@ -942,6 +1008,9 @@ { "spawn": "integ:ondemand-googleanalytics4-to-s3:assert" }, + { + "spawn": "integ:ondemand-microsoftsharepointonline-to-s3:assert" + }, { "spawn": "integ:ondemand-s3-to-snowflake:assert" }, diff --git a/API.md b/API.md index 46b63f76..9f89afad 100644 --- a/API.md +++ b/API.md @@ -1297,6 +1297,10 @@ public readonly credentials: ISecret; ### MicrosoftSharepointOnlineConnectorProfile +A class that represents a Microsoft Sharepoint Online Connector Profile. + +This connector profile allows to transfer document libraries residing on a Microsoft Sharepoint Online's site to Amazon S3. + #### Initializers ```typescript @@ -6160,13 +6164,27 @@ const microsoftSharepointOnlineOAuthSettings: MicrosoftSharepointOnlineOAuthSett | **Name** | **Type** | **Description** | | --- | --- | --- | +| accessToken | string | The access token to be used when interacting with Microsoft Sharepoint Online. | | endpoints | MicrosoftSharepointOnlineOAuthEndpointsSettings | *No description.* | -| accessToken | string | The access token to be used when interacting with Google Analytics 4. | | flow | MicrosoftSharepointOnlineOAuthFlow | *No description.* | --- -##### `endpoints`Required +##### `accessToken`Optional + +```typescript +public readonly accessToken: string; +``` + +- *Type:* string + +The access token to be used when interacting with Microsoft Sharepoint Online. + +Note that if only the access token is provided AppFlow is not able to retrieve a fresh access token when the current one is expired + +--- + +##### `endpoints`Optional ```typescript public readonly endpoints: MicrosoftSharepointOnlineOAuthEndpointsSettings; @@ -6176,27 +6194,62 @@ public readonly endpoints: MicrosoftSharepointOnlineOAuthEndpointsSettings; --- -##### `accessToken`Optional +##### `flow`Optional ```typescript -public readonly accessToken: string; +public readonly flow: MicrosoftSharepointOnlineOAuthFlow; ``` -- *Type:* string +- *Type:* MicrosoftSharepointOnlineOAuthFlow -The access token to be used when interacting with Google Analytics 4. +--- -Note that if only the access token is provided AppFlow is not able to retrieve a fresh access token when the current one is expired +### MicrosoftSharepointOnlineObject + +Represents a list of Microsoft Sharepoint Online site drives from which to retrieve the documents. + +#### Initializer + +```typescript +import { MicrosoftSharepointOnlineObject } from '@cdklabs/cdk-appflow' + +const microsoftSharepointOnlineObject: MicrosoftSharepointOnlineObject = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| drives | string[] | An array of Microsoft Sharepoint Online site drives from which the documents are to be retrieved. | +| site | string | The Microsoft Sharepoint Online site from which the documents are to be retrieved. | --- -##### `flow`Optional +##### `drives`Required ```typescript -public readonly flow: MicrosoftSharepointOnlineOAuthFlow; +public readonly drives: string[]; ``` -- *Type:* MicrosoftSharepointOnlineOAuthFlow +- *Type:* string[] + +An array of Microsoft Sharepoint Online site drives from which the documents are to be retrieved. + +Note: each drive requires full name starting with 'drives/' + +--- + +##### `site`Required + +```typescript +public readonly site: string; +``` + +- *Type:* string + +The Microsoft Sharepoint Online site from which the documents are to be retrieved. + +Note: requires full name starting with 'sites/' --- @@ -6252,7 +6305,7 @@ public readonly refreshToken: string; ### MicrosoftSharepointOnlineSourceProps -Properties of a Google Analytics v4 Source. +Properties of a Microsoft Sharepoint Online Source. #### Initializer @@ -6267,7 +6320,7 @@ const microsoftSharepointOnlineSourceProps: MicrosoftSharepointOnlineSourceProps | **Name** | **Type** | **Description** | | --- | --- | --- | | apiVersion | string | *No description.* | -| object | string | *No description.* | +| object | MicrosoftSharepointOnlineObject | *No description.* | | profile | MicrosoftSharepointOnlineConnectorProfile | *No description.* | --- @@ -6285,10 +6338,10 @@ public readonly apiVersion: string; ##### `object`Required ```typescript -public readonly object: string; +public readonly object: MicrosoftSharepointOnlineObject; ``` -- *Type:* string +- *Type:* MicrosoftSharepointOnlineObject --- @@ -10823,7 +10876,7 @@ The AppFlow type of the connector that this source is implemented for. - *Implements:* ISource -A class that represents a Google Analytics v4 Source. +A class that represents a Microsoft Sharepoint Online Source. #### Initializers @@ -10889,6 +10942,8 @@ The AppFlow type of the connector that this source is implemented for. ### MicrosoftSharepointOnlineTokenUrlBuilder +A utility class for building Microsoft Online token URLs. + #### Initializers ```typescript @@ -10907,19 +10962,19 @@ new MicrosoftSharepointOnlineTokenUrlBuilder() | **Name** | **Description** | | --- | --- | -| buildFromTenant | *No description.* | +| buildTokenUrl | *No description.* | --- -##### `buildFromTenant` +##### `buildTokenUrl` ```typescript import { MicrosoftSharepointOnlineTokenUrlBuilder } from '@cdklabs/cdk-appflow' -MicrosoftSharepointOnlineTokenUrlBuilder.buildFromTenant(tenantId: string) +MicrosoftSharepointOnlineTokenUrlBuilder.buildTokenUrl(tenantId?: string) ``` -###### `tenantId`Required +###### `tenantId`Optional - *Type:* string @@ -13233,16 +13288,20 @@ The AppFlow type of the connector that this source is implemented for. ### MicrosoftSharepointOnlineApiVersion +An enum representing the Microsoft Sharepoint Online API versions. + #### Members | **Name** | **Description** | | --- | --- | -| V1 | *No description.* | +| V1 | Version 1.0. | --- ##### `V1` +Version 1.0. + --- diff --git a/package.json b/package.json index 55c93030..7b5f564b 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,11 @@ "integ:ondemand-googleanalytics4-to-s3:destroy": "npx projen integ:ondemand-googleanalytics4-to-s3:destroy", "integ:ondemand-googleanalytics4-to-s3:snapshot": "npx projen integ:ondemand-googleanalytics4-to-s3:snapshot", "integ:ondemand-googleanalytics4-to-s3:watch": "npx projen integ:ondemand-googleanalytics4-to-s3:watch", + "integ:ondemand-microsoftsharepointonline-to-s3:assert": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:assert", + "integ:ondemand-microsoftsharepointonline-to-s3:deploy": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:deploy", + "integ:ondemand-microsoftsharepointonline-to-s3:destroy": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:destroy", + "integ:ondemand-microsoftsharepointonline-to-s3:snapshot": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:snapshot", + "integ:ondemand-microsoftsharepointonline-to-s3:watch": "npx projen integ:ondemand-microsoftsharepointonline-to-s3:watch", "integ:ondemand-s3-to-snowflake:assert": "npx projen integ:ondemand-s3-to-snowflake:assert", "integ:ondemand-s3-to-snowflake:deploy": "npx projen integ:ondemand-s3-to-snowflake:deploy", "integ:ondemand-s3-to-snowflake:destroy": "npx projen integ:ondemand-s3-to-snowflake:destroy", diff --git a/src/microsoftsharepointonline/profile.ts b/src/microsoftsharepointonline/profile.ts index 235a963c..8fdc1e8c 100644 --- a/src/microsoftsharepointonline/profile.ts +++ b/src/microsoftsharepointonline/profile.ts @@ -5,6 +5,7 @@ SPDX-License-Identifier: Apache-2.0 import { CfnConnectorProfile } from 'aws-cdk-lib/aws-appflow'; import { Construct } from 'constructs'; import { MicrosoftSharepointOnlineConnectorType } from './type'; +import { MicrosoftSharepointOnlineTokenUrlBuilder } from './util'; import { ConnectorAuthenticationType } from '../core/connectors/connector-authentication-type'; import { ConnectorProfileBase, ConnectorProfileProps } from '../core/connectors/connector-profile'; import { OAuth2GrantType as OAuthGrantType } from '../core/connectors/oauth2-granttype'; @@ -30,7 +31,7 @@ export interface MicrosoftSharepointOnlineOAuthFlow { export interface MicrosoftSharepointOnlineOAuthSettings { /** - * The access token to be used when interacting with Google Analytics 4 + * The access token to be used when interacting with Microsoft Sharepoint Online * * Note that if only the access token is provided AppFlow is not able to retrieve a fresh access token when the current one is expired */ @@ -38,9 +39,14 @@ export interface MicrosoftSharepointOnlineOAuthSettings { readonly flow?: MicrosoftSharepointOnlineOAuthFlow; - readonly endpoints: MicrosoftSharepointOnlineOAuthEndpointsSettings; + readonly endpoints?: MicrosoftSharepointOnlineOAuthEndpointsSettings; } +/** + * A class that represents a Microsoft Sharepoint Online Connector Profile. + * + * This connector profile allows to transfer document libraries residing on a Microsoft Sharepoint Online's site to Amazon S3. + */ export class MicrosoftSharepointOnlineConnectorProfile extends ConnectorProfileBase { public static fromConnectionProfileArn(scope: Construct, id: string, arn: string) { @@ -51,6 +57,8 @@ export class MicrosoftSharepointOnlineConnectorProfile extends ConnectorProfileB return this._fromConnectorProfileAttributes(scope, id, { name }) as MicrosoftSharepointOnlineConnectorProfile; } + private static readonly defaultTokenEndpoint = MicrosoftSharepointOnlineTokenUrlBuilder.buildTokenUrl(); + constructor(scope: Construct, id: string, props: MicrosoftSharepointOnlineConnectorProfileProps) { super(scope, id, props, MicrosoftSharepointOnlineConnectorType.instance); } @@ -66,7 +74,7 @@ export class MicrosoftSharepointOnlineConnectorProfile extends ConnectorProfileB oAuth2GrantType: OAuthGrantType.AUTHORIZATION_CODE, // INFO: even if we provide only the access token this property is required // TODO: think about if this is correct. token can be IResolvable - tokenUrl: properties.oAuth.endpoints?.token, + tokenUrl: properties.oAuth.endpoints?.token ?? MicrosoftSharepointOnlineConnectorProfile.defaultTokenEndpoint, }, }, }; diff --git a/src/microsoftsharepointonline/source.ts b/src/microsoftsharepointonline/source.ts index 7d9d3ffa..cf5029d9 100644 --- a/src/microsoftsharepointonline/source.ts +++ b/src/microsoftsharepointonline/source.ts @@ -2,6 +2,7 @@ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 */ +import { Fn } from 'aws-cdk-lib'; import { CfnFlow } from 'aws-cdk-lib/aws-appflow'; import { IConstruct } from 'constructs'; import { MicrosoftSharepointOnlineConnectorProfile } from './profile'; @@ -11,16 +12,35 @@ import { IFlow } from '../core/flows'; import { ISource } from '../core/vertices'; /** - * Properties of a Google Analytics v4 Source + * Represents a list of Microsoft Sharepoint Online site drives from which to retrieve the documents. + */ +export interface MicrosoftSharepointOnlineObject { + /** + * The Microsoft Sharepoint Online site from which the documents are to be retrieved. + * + * Note: requires full name starting with 'sites/' + */ + readonly site: string; + + /** + * An array of Microsoft Sharepoint Online site drives from which the documents are to be retrieved. + * + * Note: each drive requires full name starting with 'drives/' + */ + readonly drives: string[]; +} + +/** + * Properties of a Microsoft Sharepoint Online Source */ export interface MicrosoftSharepointOnlineSourceProps { readonly profile: MicrosoftSharepointOnlineConnectorProfile; readonly apiVersion: string; - readonly object: string; + readonly object: MicrosoftSharepointOnlineObject; } /** - * A class that represents a Google Analytics v4 Source + * A class that represents a Microsoft Sharepoint Online Source */ export class MicrosoftSharepointOnlineSource implements ISource { @@ -44,9 +64,17 @@ export class MicrosoftSharepointOnlineSource implements ISource { } private buildSourceConnectorProperties(): CfnFlow.SourceConnectorPropertiesProperty { + + if (this.props.object.drives.length < 1) { + throw new Error('At least one drive must be specified'); + } + return { customConnector: { - entityName: this.props.object, + entityName: this.props.object.site, + customProperties: { + subEntities: `["${Fn.join('","', this.props.object.drives)}"]`, + }, }, }; } diff --git a/src/microsoftsharepointonline/type.ts b/src/microsoftsharepointonline/type.ts index c5dc791a..b3f66aca 100644 --- a/src/microsoftsharepointonline/type.ts +++ b/src/microsoftsharepointonline/type.ts @@ -19,6 +19,6 @@ export class MicrosoftSharepointOnlineConnectorType extends ConnectorType { private static actualInstance: ConnectorType; constructor() { - super('MS_SharePoint_Connector', true); + super('MicrosoftSharePointOnline', true); } } \ No newline at end of file diff --git a/src/microsoftsharepointonline/util.ts b/src/microsoftsharepointonline/util.ts index c56c3094..c643e034 100644 --- a/src/microsoftsharepointonline/util.ts +++ b/src/microsoftsharepointonline/util.ts @@ -2,12 +2,22 @@ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 */ + +/** + * A utility class for building Microsoft Online token URLs. + */ export class MicrosoftSharepointOnlineTokenUrlBuilder { - public static buildFromTenant(tenantId: string) { - return `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`; + public static buildTokenUrl(tenantId?: string) { + return `https://login.microsoftonline.com/${tenantId ?? 'common'}/oauth2/v2.0/token`; } } +/** + * An enum representing the Microsoft Sharepoint Online API versions. + */ export enum MicrosoftSharepointOnlineApiVersion { + /** + * Version 1.0 + */ V1 = 'v1.0' } \ No newline at end of file diff --git a/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.assets.json new file mode 100644 index 00000000..eff0f0b3 --- /dev/null +++ b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.assets.json @@ -0,0 +1,32 @@ +{ + "version": "32.0.0", + "files": { + "350185a1069fa20a23a583e20c77f6844218bd73097902362dc94f1a108f5d89": { + "source": { + "path": "asset.350185a1069fa20a23a583e20c77f6844218bd73097902362dc94f1a108f5d89", + "packaging": "zip" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "350185a1069fa20a23a583e20c77f6844218bd73097902362dc94f1a108f5d89.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "6b57b7d92e5a078619ebade6f916f2f97a85517dd96d351c12af4d3b69ccaa6c": { + "source": { + "path": "TestStack.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "6b57b7d92e5a078619ebade6f916f2f97a85517dd96d351c12af4d3b69ccaa6c.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.template.json b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.template.json new file mode 100644 index 00000000..6e9cc8ba --- /dev/null +++ b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.template.json @@ -0,0 +1,509 @@ +{ + "Resources": { + "TestConnectorEE9F0A13": { + "Type": "AWS::AppFlow::ConnectorProfile", + "Properties": { + "ConnectionMode": "Public", + "ConnectorProfileName": "TestConnector", + "ConnectorType": "CustomConnector", + "ConnectorLabel": "MicrosoftSharePointOnline", + "ConnectorProfileConfig": { + "ConnectorProfileCredentials": { + "CustomConnector": { + "AuthenticationType": "OAUTH2", + "Oauth2": { + "AccessToken": { + "Fn::Join": [ + "", + [ + "{{resolve:secretsmanager:arn:", + { + "Ref": "AWS::Partition" + }, + ":secretsmanager:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":secret:appflow/mssharepointonline:SecretString:accessToken::}}" + ] + ] + }, + "RefreshToken": { + "Fn::Join": [ + "", + [ + "{{resolve:secretsmanager:arn:", + { + "Ref": "AWS::Partition" + }, + ":secretsmanager:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":secret:appflow/mssharepointonline:SecretString:refreshToken::}}" + ] + ] + } + } + } + }, + "ConnectorProfileProperties": { + "CustomConnector": { + "OAuth2Properties": { + "OAuth2GrantType": "AUTHORIZATION_CODE", + "TokenUrl": { + "Fn::Join": [ + "", + [ + "https://login.microsoftonline.com/{{resolve:secretsmanager:arn:", + { + "Ref": "AWS::Partition" + }, + ":secretsmanager:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":secret:appflow/mssharepointonline:SecretString:tenantId::}}/oauth2/v2.0/token" + ] + ] + } + } + } + } + } + } + }, + "TestBucket560B80BC": { + "Type": "AWS::S3::Bucket", + "Properties": { + "Tags": [ + { + "Key": "aws-cdk:auto-delete-objects", + "Value": "true" + } + ] + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "TestBucketPolicyBA12ED38": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "TestBucket560B80BC" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "TestBucket560B80BC", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "TestBucket560B80BC", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:PutObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:ListBucketMultipartUploads", + "s3:GetBucketAcl", + "s3:PutObjectAcl" + ], + "Condition": { + "StringEquals": { + "aws:SourceAccount": { + "Ref": "AWS::AccountId" + } + } + }, + "Effect": "Allow", + "Principal": { + "Service": "appflow.amazonaws.com" + }, + "Resource": [ + { + "Fn::GetAtt": [ + "TestBucket560B80BC", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "TestBucket560B80BC", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "TestBucketAutoDeleteObjectsCustomResource8FEAABD5": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "TestBucket560B80BC" + } + }, + "DependsOn": [ + "TestBucketPolicyBA12ED38" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ] + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + } + ] + } + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "350185a1069fa20a23a583e20c77f6844218bd73097902362dc94f1a108f5d89.zip" + }, + "Timeout": 900, + "MemorySize": 128, + "Handler": "__entrypoint__.handler", + "Role": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + }, + "Runtime": "nodejs16.x", + "Description": { + "Fn::Join": [ + "", + [ + "Lambda function for auto-deleting objects in ", + { + "Ref": "TestBucket560B80BC" + }, + " S3 bucket." + ] + ] + } + }, + "DependsOn": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" + ] + }, + "OnDemandFlow4ECA54C5": { + "Type": "AWS::AppFlow::Flow", + "Properties": { + "DestinationFlowConfigList": [ + { + "ConnectorType": "S3", + "DestinationConnectorProperties": { + "S3": { + "BucketName": { + "Ref": "TestBucket560B80BC" + }, + "S3OutputFormatConfig": { + "AggregationConfig": { + "AggregationType": "SingleFile" + }, + "FileType": "JSON" + } + } + } + } + ], + "FlowName": "OnDemandFlow", + "SourceFlowConfig": { + "ApiVersion": "v1.0", + "ConnectorProfileName": "TestConnector", + "ConnectorType": "CustomConnector", + "SourceConnectorProperties": { + "CustomConnector": { + "CustomProperties": { + "subEntities": { + "Fn::Join": [ + "", + [ + "[\"{{resolve:secretsmanager:arn:", + { + "Ref": "AWS::Partition" + }, + ":secretsmanager:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":secret:appflow/mssharepointonline:SecretString:drive::}}\"]" + ] + ] + } + }, + "EntityName": { + "Fn::Join": [ + "", + [ + "{{resolve:secretsmanager:arn:", + { + "Ref": "AWS::Partition" + }, + ":secretsmanager:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":secret:appflow/mssharepointonline:SecretString:site::}}" + ] + ] + } + } + } + }, + "Tasks": [ + { + "ConnectorOperator": { + "CustomConnector": "NO_OP" + }, + "SourceFields": [], + "TaskProperties": [ + { + "Key": "EXCLUDE_SOURCE_FIELDS_LIST", + "Value": "[]" + } + ], + "TaskType": "Map_all" + } + ], + "TriggerConfig": { + "TriggerType": "OnDemand" + } + }, + "DependsOn": [ + "TestBucketAutoDeleteObjectsCustomResource8FEAABD5", + "TestBucketPolicyBA12ED38", + "TestBucket560B80BC", + "TestConnectorEE9F0A13" + ] + } + }, + "Mappings": { + "DefaultCrNodeVersionMap": { + "af-south-1": { + "value": "nodejs16.x" + }, + "ap-east-1": { + "value": "nodejs16.x" + }, + "ap-northeast-1": { + "value": "nodejs16.x" + }, + "ap-northeast-2": { + "value": "nodejs16.x" + }, + "ap-northeast-3": { + "value": "nodejs16.x" + }, + "ap-south-1": { + "value": "nodejs16.x" + }, + "ap-south-2": { + "value": "nodejs16.x" + }, + "ap-southeast-1": { + "value": "nodejs16.x" + }, + "ap-southeast-2": { + "value": "nodejs16.x" + }, + "ap-southeast-3": { + "value": "nodejs16.x" + }, + "ca-central-1": { + "value": "nodejs16.x" + }, + "cn-north-1": { + "value": "nodejs16.x" + }, + "cn-northwest-1": { + "value": "nodejs16.x" + }, + "eu-central-1": { + "value": "nodejs16.x" + }, + "eu-central-2": { + "value": "nodejs16.x" + }, + "eu-north-1": { + "value": "nodejs16.x" + }, + "eu-south-1": { + "value": "nodejs16.x" + }, + "eu-south-2": { + "value": "nodejs16.x" + }, + "eu-west-1": { + "value": "nodejs16.x" + }, + "eu-west-2": { + "value": "nodejs16.x" + }, + "eu-west-3": { + "value": "nodejs16.x" + }, + "me-central-1": { + "value": "nodejs16.x" + }, + "me-south-1": { + "value": "nodejs16.x" + }, + "sa-east-1": { + "value": "nodejs16.x" + }, + "us-east-1": { + "value": "nodejs16.x" + }, + "us-east-2": { + "value": "nodejs16.x" + }, + "us-gov-east-1": { + "value": "nodejs16.x" + }, + "us-gov-west-1": { + "value": "nodejs16.x" + }, + "us-iso-east-1": { + "value": "nodejs14.x" + }, + "us-iso-west-1": { + "value": "nodejs14.x" + }, + "us-isob-east-1": { + "value": "nodejs14.x" + }, + "us-west-1": { + "value": "nodejs16.x" + }, + "us-west-2": { + "value": "nodejs16.x" + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts new file mode 100644 index 00000000..9bb06cbe --- /dev/null +++ b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.ts @@ -0,0 +1,61 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { App, RemovalPolicy, Stack } from 'aws-cdk-lib'; +import { Bucket } from 'aws-cdk-lib/aws-s3'; +import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; +import { MicrosoftSharepointOnlineApiVersion, MicrosoftSharepointOnlineConnectorProfile, MicrosoftSharepointOnlineSource, Mapping, OnDemandFlow, S3Destination, MicrosoftSharepointOnlineTokenUrlBuilder, S3OutputAggregationType } from '../../src'; + +const app = new App({ + treeMetadata: false, +}); + +const stack = new Stack(app, 'TestStack'); + +const secret = Secret.fromSecretNameV2(stack, 'TestSecret', 'appflow/mssharepointonline'); + +const profile = new MicrosoftSharepointOnlineConnectorProfile(stack, 'TestConnector', { + oAuth: { + accessToken: secret.secretValueFromJson('accessToken').toString(), + flow: { + refreshTokenGrant: { + refreshToken: secret.secretValueFromJson('refreshToken').toString(), + }, + }, + endpoints: { + token: MicrosoftSharepointOnlineTokenUrlBuilder.buildTokenUrl(secret.secretValueFromJson('tenantId').toString()), + }, + }, +}); + +const source = new MicrosoftSharepointOnlineSource({ + profile: profile, + apiVersion: MicrosoftSharepointOnlineApiVersion.V1, + object: { + site: secret.secretValueFromJson('site').toString(), + drives: [secret.secretValueFromJson('drive').toString()], + }, +}); + +const bucket = new Bucket(stack, 'TestBucket', { + autoDeleteObjects: true, + removalPolicy: RemovalPolicy.DESTROY, +}); + +const destination = new S3Destination({ + location: { bucket }, + formatting: { + aggregation: { + type: S3OutputAggregationType.SINGLE_FILE, + }, + }, +}); + +new OnDemandFlow(stack, 'OnDemandFlow', { + source: source, + destination: destination, + mappings: [Mapping.mapAll()], +}); + +app.synth(); diff --git a/test/microsoftsharepointonline/profile.test.ts b/test/microsoftsharepointonline/profile.test.ts index 26b55b74..479bdff6 100644 --- a/test/microsoftsharepointonline/profile.test.ts +++ b/test/microsoftsharepointonline/profile.test.ts @@ -36,7 +36,7 @@ describe('MicrosoftSharepointOnlineConnectorProfile', () => { ConnectionMode: 'Public', ConnectorProfileName: 'TestProfile', ConnectorType: 'CustomConnector', - ConnectorLabel: 'MS_SharePoint_Connector', + ConnectorLabel: 'MicrosoftSharePointOnline', ConnectorProfileConfig: { ConnectorProfileCredentials: { CustomConnector: { @@ -86,7 +86,7 @@ describe('MicrosoftSharepointOnlineConnectorProfile', () => { ConnectionMode: 'Public', ConnectorProfileName: 'TestProfile', ConnectorType: 'CustomConnector', - ConnectorLabel: 'MS_SharePoint_Connector', + ConnectorLabel: 'MicrosoftSharePointOnline', ConnectorProfileConfig: { ConnectorProfileCredentials: { CustomConnector: { @@ -196,7 +196,7 @@ describe('MicrosoftSharepointOnlineConnectorProfile', () => { ConnectionMode: 'Public', ConnectorProfileName: 'TestProfile', ConnectorType: 'CustomConnector', - ConnectorLabel: 'MS_SharePoint_Connector', + ConnectorLabel: 'MicrosoftSharePointOnline', ConnectorProfileConfig: { ConnectorProfileCredentials: { CustomConnector: { diff --git a/test/microsoftsharepointonline/source.test.ts b/test/microsoftsharepointonline/source.test.ts index 14201b51..b443fde2 100644 --- a/test/microsoftsharepointonline/source.test.ts +++ b/test/microsoftsharepointonline/source.test.ts @@ -22,7 +22,10 @@ describe('MicrosoftSharepointOnlineSource', () => { const stack = new Stack(undefined, 'TestStack'); const source = new MicrosoftSharepointOnlineSource({ profile: MicrosoftSharepointOnlineConnectorProfile.fromConnectionProfileName(stack, 'TestProfile', 'dummy-profile'), - object: 'dummy-object', + object: { + site: 'sites/dummysite.sharepoint.com,3f42g340-bc23-4a31-b7e5-722e57c39cb8,5bbc39fb-2b17-423b-a007-40ca508389a5', + drives: ['drives/b!fcPDltwTLSougEJuDFjE?U5qHuXbkzlvSaA5oNoMW4tB0y6mebcx9m-ckwA9KtKE'], + }, apiVersion: MicrosoftSharepointOnlineApiVersion.V1, }); @@ -37,7 +40,10 @@ describe('MicrosoftSharepointOnlineSource', () => { const stack = new Stack(undefined, 'TestStack'); const source = new MicrosoftSharepointOnlineSource({ profile: MicrosoftSharepointOnlineConnectorProfile.fromConnectionProfileName(stack, 'TestProfile', 'dummy-profile'), - object: 'dummy-object', + object: { + site: 'sites/dummysite.sharepoint.com,3f42g340-bc23-4a31-b7e5-722e57c39cb8,5bbc39fb-2b17-423b-a007-40ca508389a5', + drives: ['drives/b!fcPDltwTLSougEJuDFjE?U5qHuXbkzlvSaA5oNoMW4tB0y6mebcx9m-ckwA9KtKE'], + }, apiVersion: MicrosoftSharepointOnlineApiVersion.V1, }); @@ -71,7 +77,10 @@ describe('MicrosoftSharepointOnlineSource', () => { ConnectorType: 'CustomConnector', SourceConnectorProperties: { CustomConnector: { - EntityName: 'dummy-object', + EntityName: 'sites/dummysite.sharepoint.com,3f42g340-bc23-4a31-b7e5-722e57c39cb8,5bbc39fb-2b17-423b-a007-40ca508389a5', + CustomProperties: { + subEntities: '["drives/b!fcPDltwTLSougEJuDFjE?U5qHuXbkzlvSaA5oNoMW4tB0y6mebcx9m-ckwA9KtKE"]', + }, }, }, }, @@ -117,7 +126,10 @@ describe('MicrosoftSharepointOnlineSource', () => { const source = new MicrosoftSharepointOnlineSource({ profile: profile, - object: 'dummy-object', + object: { + site: 'sites/dummysite.sharepoint.com,3f42g340-bc23-4a31-b7e5-722e57c39cb8,5bbc39fb-2b17-423b-a007-40ca508389a5', + drives: ['drives/b!fcPDltwTLSougEJuDFjE?U5qHuXbkzlvSaA5oNoMW4tB0y6mebcx9m-ckwA9KtKE'], + }, apiVersion: MicrosoftSharepointOnlineApiVersion.V1, }); @@ -137,7 +149,7 @@ describe('MicrosoftSharepointOnlineSource', () => { ConnectionMode: 'Public', ConnectorProfileName: 'TestProfile', ConnectorType: 'CustomConnector', - ConnectorLabel: 'MS_SharePoint_Connector', + ConnectorLabel: 'MicrosoftSharePointOnline', ConnectorProfileConfig: { ConnectorProfileCredentials: { CustomConnector: { @@ -182,7 +194,10 @@ describe('MicrosoftSharepointOnlineSource', () => { ConnectorType: 'CustomConnector', SourceConnectorProperties: { CustomConnector: { - EntityName: 'dummy-object', + EntityName: 'sites/dummysite.sharepoint.com,3f42g340-bc23-4a31-b7e5-722e57c39cb8,5bbc39fb-2b17-423b-a007-40ca508389a5', + CustomProperties: { + subEntities: '["drives/b!fcPDltwTLSougEJuDFjE?U5qHuXbkzlvSaA5oNoMW4tB0y6mebcx9m-ckwA9KtKE"]', + }, }, }, },