From d2ef4ca86d107505141ea3c36f731a80fb18e6c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Konrad=20Paw=C5=82aszek?= Date: Wed, 17 Jan 2024 16:45:24 +0100 Subject: [PATCH] feat: adding Google BigQuery components (#82) This PR: - adds [Google Big Query Connector Profile](https://docs.aws.amazon.com/appflow/latest/userguide/connectors-googlebigquery.html) - adds Google Big Query Flow source - bumps the version of AWS CDK to 2.121.1 - upgrades the CDK Labs deps --- .github/workflows/release.yml | 19 +- .gitignore | 2 +- .npmignore | 2 + .projen/deps.json | 6 +- .projen/tasks.json | 3 +- .projenrc.ts | 2 +- API.md | 900 ++++++++++++++++-- package.json | 16 +- src/googlebigquery/index.ts | 8 + src/googlebigquery/profile.ts | 137 +++ src/googlebigquery/source.ts | 65 ++ src/googlebigquery/type.ts | 24 + src/googlebigquery/util.ts | 7 + src/index.ts | 1 + test/googlebigquery/profile.test.ts | 286 ++++++ test/googlebigquery/source.test.ts | 224 +++++ .../TestStack.assets.json | 19 - .../TestStack.template.json | 165 ---- .../cdk.out | 1 - .../TestStack.assets.json | 2 +- .../TestStack.assets.json | 2 +- .../cdk.out | 1 + .../manifest.json | 21 +- .../TestStack.assets.json | 2 +- .../cdk.out | 1 + .../manifest.json | 95 ++ .../TestStack.assets.json | 2 +- .../index.js | 1 + .../cdk.out | 1 + .../manifest.json | 95 ++ .../TestStack.assets.json | 2 +- .../TestStack.assets.json | 2 +- .../TestStack.assets.json | 18 +- .../TestStack.template.json | 4 +- .../TestStack.assets.json | 2 +- .../TestStack.assets.json | 2 +- .../TestStack.assets.json | 2 +- .../TestStack.assets.json | 2 +- .../TestStack.assets.json | 2 +- .../TestStack.assets.json | 2 +- tsconfig.dev.json | 1 - yarn.lock | 40 +- 42 files changed, 1840 insertions(+), 349 deletions(-) create mode 100644 src/googlebigquery/index.ts create mode 100644 src/googlebigquery/profile.ts create mode 100644 src/googlebigquery/source.ts create mode 100644 src/googlebigquery/type.ts create mode 100644 src/googlebigquery/util.ts create mode 100644 test/googlebigquery/profile.test.ts create mode 100644 test/googlebigquery/source.test.ts delete mode 100644 test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/TestStack.assets.json delete mode 100644 test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/TestStack.template.json delete mode 100644 test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/cdk.out create mode 100644 test/integ/ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot/cdk.out rename test/integ/{ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot => ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot}/manifest.json (72%) create mode 100644 test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/cdk.out create mode 100644 test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/manifest.json create mode 100644 test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/asset.b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6/index.js create mode 100644 test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/cdk.out create mode 100644 test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/manifest.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 916aa39e..9a2e5fb5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,7 @@ jobs: contents: write outputs: latest_commit: ${{ steps.git_remote.outputs.latest_commit }} + tag_exists: ${{ steps.check_tag_exists.outputs.exists }} env: CI: "true" steps: @@ -32,6 +33,12 @@ jobs: run: yarn install --check-files --frozen-lockfile - name: release run: npx projen release + - name: Check if releasetag already exists + id: check_tag_exists + run: |- + TAG=$(cat dist/dist/releasetag.txt) + ([ ! -z "$TAG" ] && git ls-remote -q --exit-code --tags origin $TAG && (echo "exists=true" >> $GITHUB_OUTPUT)) || echo "exists=false" >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT - name: Check for new commits id: git_remote run: echo "latest_commit=$(git ls-remote origin -h ${{ github.ref }} | cut -f1)" >> $GITHUB_OUTPUT @@ -51,7 +58,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-node@v3 with: @@ -80,7 +87,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-node@v3 with: @@ -113,7 +120,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-java@v3 with: @@ -153,7 +160,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-node@v3 with: @@ -188,7 +195,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-node@v3 with: @@ -222,7 +229,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-node@v3 with: diff --git a/.gitignore b/.gitignore index 7525aee2..1df3781d 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ jspm_packages/ *.rest .vscode **/.DS_Store -!/.projenrc.js /test-reports/ junit.xml /coverage/ @@ -165,3 +164,4 @@ test/integ/onschedule-salesforce-marketing-cloud-to-s3.integ.snapshot/**/tree.js !/.github/workflows/upgrade-cdklabs-projen-project-types-main.yml !/.github/workflows/upgrade-main.yml !/.github/workflows/upgrade-dev-deps-main.yml +!/.projenrc.ts diff --git a/.npmignore b/.npmignore index 7b7315e8..fdaa448c 100644 --- a/.npmignore +++ b/.npmignore @@ -37,3 +37,5 @@ test/integ/onevent-salesforce-to-eventbridge.integ.snapshot test/integ/onschedule-s3-to-salesforce.integ.snapshot test/integ/.tmp test/integ/onschedule-salesforce-marketing-cloud-to-s3.integ.snapshot +/.gitattributes +/.projenrc.ts diff --git a/.projen/deps.json b/.projen/deps.json index a9066327..82f07c44 100644 --- a/.projen/deps.json +++ b/.projen/deps.json @@ -112,17 +112,17 @@ }, { "name": "@aws-cdk/aws-glue-alpha", - "version": "2.116.1-alpha.0", + "version": "2.121.1-alpha.0", "type": "peer" }, { "name": "@aws-cdk/aws-redshift-alpha", - "version": "2.116.1-alpha.0", + "version": "2.121.1-alpha.0", "type": "peer" }, { "name": "aws-cdk-lib", - "version": "^2.116.1", + "version": "^2.121.1", "type": "peer" }, { diff --git a/.projen/tasks.json b/.projen/tasks.json index 7ba967ab..d5e52525 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -154,7 +154,8 @@ "description": "Runs eslint against the codebase", "steps": [ { - "exec": "eslint --ext .ts,.tsx --fix --no-error-on-unmatched-pattern src test build-tools projenrc .projenrc.ts" + "exec": "eslint --ext .ts,.tsx --fix --no-error-on-unmatched-pattern $@ src test build-tools projenrc .projenrc.ts", + "receiveArgs": true } ] }, diff --git a/.projenrc.ts b/.projenrc.ts index 9a7302fd..9b54eb97 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 import { CdklabsConstructLibrary } from 'cdklabs-projen-project-types'; import { Stability } from 'projen/lib/cdk'; -const cdkVersion = '2.116.1'; +const cdkVersion = '2.121.1'; const project = new CdklabsConstructLibrary({ name: '@cdklabs/cdk-appflow', author: 'Amazon Web Services', diff --git a/API.md b/API.md index 1f3fd1c4..7e69e6e6 100644 --- a/API.md +++ b/API.md @@ -1447,6 +1447,282 @@ public readonly credentials: ISecret; --- +### GoogleBigQueryConnectorProfile + +#### Initializers + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +new GoogleBigQueryConnectorProfile(scope: Construct, id: string, props: GoogleBigQueryConnectorProfileProps) +``` + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| scope | constructs.Construct | *No description.* | +| id | string | *No description.* | +| props | GoogleBigQueryConnectorProfileProps | *No description.* | + +--- + +##### `scope`Required + +- *Type:* constructs.Construct + +--- + +##### `id`Required + +- *Type:* string + +--- + +##### `props`Required + +- *Type:* GoogleBigQueryConnectorProfileProps + +--- + +#### Methods + +| **Name** | **Description** | +| --- | --- | +| toString | Returns a string representation of this construct. | +| applyRemovalPolicy | Apply the given removal policy to this resource. | + +--- + +##### `toString` + +```typescript +public toString(): string +``` + +Returns a string representation of this construct. + +##### `applyRemovalPolicy` + +```typescript +public applyRemovalPolicy(policy: RemovalPolicy): void +``` + +Apply the given removal policy to this resource. + +The Removal Policy controls what happens to this resource when it stops +being managed by CloudFormation, either because you've removed it from the +CDK application or because you've made a change that requires the resource +to be replaced. + +The resource can be deleted (`RemovalPolicy.DESTROY`), or left in your AWS +account for data recovery and cleanup later (`RemovalPolicy.RETAIN`). + +###### `policy`Required + +- *Type:* aws-cdk-lib.RemovalPolicy + +--- + +#### Static Functions + +| **Name** | **Description** | +| --- | --- | +| isConstruct | Checks if `x` is a construct. | +| isOwnedResource | Returns true if the construct was created by CDK, and false otherwise. | +| isResource | Check whether the given construct is a Resource. | +| fromConnectionProfileArn | *No description.* | +| fromConnectionProfileName | *No description.* | + +--- + +##### ~~`isConstruct`~~ + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +GoogleBigQueryConnectorProfile.isConstruct(x: any) +``` + +Checks if `x` is a construct. + +###### `x`Required + +- *Type:* any + +Any object. + +--- + +##### `isOwnedResource` + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +GoogleBigQueryConnectorProfile.isOwnedResource(construct: IConstruct) +``` + +Returns true if the construct was created by CDK, and false otherwise. + +###### `construct`Required + +- *Type:* constructs.IConstruct + +--- + +##### `isResource` + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +GoogleBigQueryConnectorProfile.isResource(construct: IConstruct) +``` + +Check whether the given construct is a Resource. + +###### `construct`Required + +- *Type:* constructs.IConstruct + +--- + +##### `fromConnectionProfileArn` + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +GoogleBigQueryConnectorProfile.fromConnectionProfileArn(scope: Construct, id: string, arn: string) +``` + +###### `scope`Required + +- *Type:* constructs.Construct + +--- + +###### `id`Required + +- *Type:* string + +--- + +###### `arn`Required + +- *Type:* string + +--- + +##### `fromConnectionProfileName` + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +GoogleBigQueryConnectorProfile.fromConnectionProfileName(scope: Construct, id: string, name: string) +``` + +###### `scope`Required + +- *Type:* constructs.Construct + +--- + +###### `id`Required + +- *Type:* string + +--- + +###### `name`Required + +- *Type:* string + +--- + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| node | constructs.Node | The tree node. | +| env | aws-cdk-lib.ResourceEnvironment | The environment this resource belongs to. | +| stack | aws-cdk-lib.Stack | The stack in which this resource is defined. | +| arn | string | *No description.* | +| name | string | *No description.* | +| credentials | aws-cdk-lib.aws_secretsmanager.ISecret | *No description.* | + +--- + +##### `node`Required + +```typescript +public readonly node: Node; +``` + +- *Type:* constructs.Node + +The tree node. + +--- + +##### `env`Required + +```typescript +public readonly env: ResourceEnvironment; +``` + +- *Type:* aws-cdk-lib.ResourceEnvironment + +The environment this resource belongs to. + +For resources that are created and managed by the CDK +(generally, those created by creating new class instances like Role, Bucket, etc.), +this is always the same as the environment of the stack they belong to; +however, for imported resources +(those obtained from static methods like fromRoleArn, fromBucketName, etc.), +that might be different than the stack they were imported into. + +--- + +##### `stack`Required + +```typescript +public readonly stack: Stack; +``` + +- *Type:* aws-cdk-lib.Stack + +The stack in which this resource is defined. + +--- + +##### `arn`Required + +```typescript +public readonly arn: string; +``` + +- *Type:* string + +--- + +##### `name`Required + +```typescript +public readonly name: string; +``` + +- *Type:* string + +--- + +##### `credentials`Optional + +```typescript +public readonly credentials: ISecret; +``` + +- *Type:* aws-cdk-lib.aws_secretsmanager.ISecret + +--- + + ### JdbcSmallDataScaleConnectorProfile The connector profile for the JDBC connector. @@ -7056,111 +7332,410 @@ const flowProps: FlowProps = { ... } public readonly destination: IDestination; ``` -- *Type:* IDestination +- *Type:* IDestination + +--- + +##### `mappings`Required + +```typescript +public readonly mappings: IMapping[]; +``` + +- *Type:* IMapping[] + +--- + +##### `source`Required + +```typescript +public readonly source: ISource; +``` + +- *Type:* ISource + +--- + +##### `description`Optional + +```typescript +public readonly description: string; +``` + +- *Type:* string + +--- + +##### `filters`Optional + +```typescript +public readonly filters: IFilter[]; +``` + +- *Type:* IFilter[] + +--- + +##### `key`Optional + +```typescript +public readonly key: IKey; +``` + +- *Type:* aws-cdk-lib.aws_kms.IKey + +--- + +##### `name`Optional + +```typescript +public readonly name: string; +``` + +- *Type:* string + +--- + +##### `transforms`Optional + +```typescript +public readonly transforms: ITransform[]; +``` + +- *Type:* ITransform[] + +--- + +##### `validations`Optional + +```typescript +public readonly validations: IValidation[]; +``` + +- *Type:* IValidation[] + +--- + +### GoogleAnalytics4ConnectorProfileProps + +#### Initializer + +```typescript +import { GoogleAnalytics4ConnectorProfileProps } from '@cdklabs/cdk-appflow' + +const googleAnalytics4ConnectorProfileProps: GoogleAnalytics4ConnectorProfileProps = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| key | aws-cdk-lib.aws_kms.IKey | TODO: think if this should be here as not all connector profiles have that. | +| name | string | *No description.* | +| oAuth | GoogleAnalytics4OAuthSettings | *No description.* | + +--- + +##### `key`Optional + +```typescript +public readonly key: IKey; +``` + +- *Type:* aws-cdk-lib.aws_kms.IKey + +TODO: think if this should be here as not all connector profiles have that. + +--- + +##### `name`Optional + +```typescript +public readonly name: string; +``` + +- *Type:* string + +--- + +##### `oAuth`Required + +```typescript +public readonly oAuth: GoogleAnalytics4OAuthSettings; +``` + +- *Type:* GoogleAnalytics4OAuthSettings + +--- + +### GoogleAnalytics4OAuthEndpoints + +Google's OAuth token and authorization endpoints. + +#### Initializer + +```typescript +import { GoogleAnalytics4OAuthEndpoints } from '@cdklabs/cdk-appflow' + +const googleAnalytics4OAuthEndpoints: GoogleAnalytics4OAuthEndpoints = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| authorization | string | The OAuth authorization endpoint URI. | +| token | string | The OAuth token endpoint URI. | + +--- + +##### `authorization`Optional + +```typescript +public readonly authorization: string; +``` + +- *Type:* string + +The OAuth authorization endpoint URI. + +--- + +##### `token`Optional + +```typescript +public readonly token: string; +``` + +- *Type:* string + +The OAuth token endpoint URI. + +--- + +### GoogleAnalytics4OAuthFlow + +Represents the OAuth flow enabled for the GA4. + +#### Initializer + +```typescript +import { GoogleAnalytics4OAuthFlow } from '@cdklabs/cdk-appflow' + +const googleAnalytics4OAuthFlow: GoogleAnalytics4OAuthFlow = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| refreshTokenGrant | GoogleAnalytics4RefreshTokenGrantFlow | The details required for executing the refresh token grant flow. | + +--- + +##### `refreshTokenGrant`Required + +```typescript +public readonly refreshTokenGrant: GoogleAnalytics4RefreshTokenGrantFlow; +``` + +- *Type:* GoogleAnalytics4RefreshTokenGrantFlow + +The details required for executing the refresh token grant flow. + +--- + +### GoogleAnalytics4OAuthSettings + +#### Initializer + +```typescript +import { GoogleAnalytics4OAuthSettings } from '@cdklabs/cdk-appflow' + +const googleAnalytics4OAuthSettings: GoogleAnalytics4OAuthSettings = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| accessToken | aws-cdk-lib.SecretValue | The access token to be used when interacting with Google Analytics 4. | +| endpoints | GoogleAnalytics4OAuthEndpoints | The OAuth token and authorization endpoints. | +| flow | GoogleAnalytics4OAuthFlow | The OAuth flow used for obtaining a new accessToken when the old is not present or expired. | + +--- + +##### `accessToken`Optional + +```typescript +public readonly accessToken: SecretValue; +``` + +- *Type:* aws-cdk-lib.SecretValue +- *Default:* Retrieves a fresh accessToken with the information in the [flow property]{@link GoogleAnalytics4OAuthSettings#flow } + +The access token to be used when interacting with Google Analytics 4. + +Note that if only the access token is provided AppFlow is not able to retrieve a fresh access token when the current one is expired + +--- + +##### `endpoints`Optional + +```typescript +public readonly endpoints: GoogleAnalytics4OAuthEndpoints; +``` + +- *Type:* GoogleAnalytics4OAuthEndpoints + +The OAuth token and authorization endpoints. + +--- + +##### `flow`Optional + +```typescript +public readonly flow: GoogleAnalytics4OAuthFlow; +``` + +- *Type:* GoogleAnalytics4OAuthFlow +- *Default:* undefined. AppFlow will not request any new accessToken after expiry. + +The OAuth flow used for obtaining a new accessToken when the old is not present or expired. --- -##### `mappings`Required +### GoogleAnalytics4RefreshTokenGrantFlow + +The OAuth elements required for the execution of the refresh token grant flow. + +#### Initializer ```typescript -public readonly mappings: IMapping[]; +import { GoogleAnalytics4RefreshTokenGrantFlow } from '@cdklabs/cdk-appflow' + +const googleAnalytics4RefreshTokenGrantFlow: GoogleAnalytics4RefreshTokenGrantFlow = { ... } ``` -- *Type:* IMapping[] +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| clientId | aws-cdk-lib.SecretValue | The id of the client app. | +| clientSecret | aws-cdk-lib.SecretValue | The secret of the client app. | +| refreshToken | aws-cdk-lib.SecretValue | A non-expired refresh token. | --- -##### `source`Required +##### `clientId`Optional ```typescript -public readonly source: ISource; +public readonly clientId: SecretValue; ``` -- *Type:* ISource +- *Type:* aws-cdk-lib.SecretValue + +The id of the client app. --- -##### `description`Optional +##### `clientSecret`Optional ```typescript -public readonly description: string; +public readonly clientSecret: SecretValue; ``` -- *Type:* string +- *Type:* aws-cdk-lib.SecretValue + +The secret of the client app. --- -##### `filters`Optional +##### `refreshToken`Optional ```typescript -public readonly filters: IFilter[]; +public readonly refreshToken: SecretValue; ``` -- *Type:* IFilter[] +- *Type:* aws-cdk-lib.SecretValue + +A non-expired refresh token. --- -##### `key`Optional +### GoogleAnalytics4SourceProps + +Properties of a Google Analytics v4 Source. + +#### Initializer ```typescript -public readonly key: IKey; +import { GoogleAnalytics4SourceProps } from '@cdklabs/cdk-appflow' + +const googleAnalytics4SourceProps: GoogleAnalytics4SourceProps = { ... } ``` -- *Type:* aws-cdk-lib.aws_kms.IKey +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| apiVersion | string | *No description.* | +| object | string | *No description.* | +| profile | GoogleAnalytics4ConnectorProfile | *No description.* | --- -##### `name`Optional +##### `apiVersion`Required ```typescript -public readonly name: string; +public readonly apiVersion: string; ``` - *Type:* string --- -##### `transforms`Optional +##### `object`Required ```typescript -public readonly transforms: ITransform[]; +public readonly object: string; ``` -- *Type:* ITransform[] +- *Type:* string --- -##### `validations`Optional +##### `profile`Required ```typescript -public readonly validations: IValidation[]; +public readonly profile: GoogleAnalytics4ConnectorProfile; ``` -- *Type:* IValidation[] +- *Type:* GoogleAnalytics4ConnectorProfile --- -### GoogleAnalytics4ConnectorProfileProps +### GoogleBigQueryConnectorProfileProps -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4ConnectorProfileProps } from '@cdklabs/cdk-appflow' +import { GoogleBigQueryConnectorProfileProps } from '@cdklabs/cdk-appflow' -const googleAnalytics4ConnectorProfileProps: GoogleAnalytics4ConnectorProfileProps = { ... } +const googleBigQueryConnectorProfileProps: GoogleBigQueryConnectorProfileProps = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| key | aws-cdk-lib.aws_kms.IKey | TODO: think if this should be here as not all connector profiles have that. | -| name | string | *No description.* | -| oAuth | GoogleAnalytics4OAuthSettings | *No description.* | +| key | aws-cdk-lib.aws_kms.IKey | TODO: think if this should be here as not all connector profiles have that. | +| name | string | *No description.* | +| oAuth | GoogleBigQueryOAuthSettings | *No description.* | --- -##### `key`Optional +##### `key`Optional ```typescript public readonly key: IKey; @@ -7172,7 +7747,7 @@ TODO: think if this should be here as not all connector profiles have that. --- -##### `name`Optional +##### `name`Optional ```typescript public readonly name: string; @@ -7182,38 +7757,38 @@ public readonly name: string; --- -##### `oAuth`Required +##### `oAuth`Required ```typescript -public readonly oAuth: GoogleAnalytics4OAuthSettings; +public readonly oAuth: GoogleBigQueryOAuthSettings; ``` -- *Type:* GoogleAnalytics4OAuthSettings +- *Type:* GoogleBigQueryOAuthSettings --- -### GoogleAnalytics4OAuthEndpoints +### GoogleBigQueryOAuthEndpoints Google's OAuth token and authorization endpoints. -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4OAuthEndpoints } from '@cdklabs/cdk-appflow' +import { GoogleBigQueryOAuthEndpoints } from '@cdklabs/cdk-appflow' -const googleAnalytics4OAuthEndpoints: GoogleAnalytics4OAuthEndpoints = { ... } +const googleBigQueryOAuthEndpoints: GoogleBigQueryOAuthEndpoints = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| authorization | string | The OAuth authorization endpoint URI. | -| token | string | The OAuth token endpoint URI. | +| authorization | string | The OAuth authorization endpoint URI. | +| token | string | The OAuth token endpoint URI. | --- -##### `authorization`Optional +##### `authorization`Optional ```typescript public readonly authorization: string; @@ -7225,7 +7800,7 @@ The OAuth authorization endpoint URI. --- -##### `token`Optional +##### `token`Optional ```typescript public readonly token: string; @@ -7237,121 +7812,171 @@ The OAuth token endpoint URI. --- -### GoogleAnalytics4OAuthFlow +### GoogleBigQueryOAuthFlow Represents the OAuth flow enabled for the GA4. -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4OAuthFlow } from '@cdklabs/cdk-appflow' +import { GoogleBigQueryOAuthFlow } from '@cdklabs/cdk-appflow' -const googleAnalytics4OAuthFlow: GoogleAnalytics4OAuthFlow = { ... } +const googleBigQueryOAuthFlow: GoogleBigQueryOAuthFlow = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| refreshTokenGrant | GoogleAnalytics4RefreshTokenGrantFlow | The details required for executing the refresh token grant flow. | +| refreshTokenGrant | GoogleBigQueryRefreshTokenGrantFlow | The details required for executing the refresh token grant flow. | --- -##### `refreshTokenGrant`Required +##### `refreshTokenGrant`Required ```typescript -public readonly refreshTokenGrant: GoogleAnalytics4RefreshTokenGrantFlow; +public readonly refreshTokenGrant: GoogleBigQueryRefreshTokenGrantFlow; ``` -- *Type:* GoogleAnalytics4RefreshTokenGrantFlow +- *Type:* GoogleBigQueryRefreshTokenGrantFlow The details required for executing the refresh token grant flow. --- -### GoogleAnalytics4OAuthSettings +### GoogleBigQueryOAuthSettings -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4OAuthSettings } from '@cdklabs/cdk-appflow' +import { GoogleBigQueryOAuthSettings } from '@cdklabs/cdk-appflow' -const googleAnalytics4OAuthSettings: GoogleAnalytics4OAuthSettings = { ... } +const googleBigQueryOAuthSettings: GoogleBigQueryOAuthSettings = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| accessToken | aws-cdk-lib.SecretValue | The access token to be used when interacting with Google Analytics 4. | -| endpoints | GoogleAnalytics4OAuthEndpoints | The OAuth token and authorization endpoints. | -| flow | GoogleAnalytics4OAuthFlow | The OAuth flow used for obtaining a new accessToken when the old is not present or expired. | +| accessToken | aws-cdk-lib.SecretValue | The access token to be used when interacting with Google BigQuery. | +| endpoints | GoogleBigQueryOAuthEndpoints | The OAuth token and authorization endpoints. | +| flow | GoogleBigQueryOAuthFlow | The OAuth flow used for obtaining a new accessToken when the old is not present or expired. | --- -##### `accessToken`Optional +##### `accessToken`Optional ```typescript public readonly accessToken: SecretValue; ``` - *Type:* aws-cdk-lib.SecretValue -- *Default:* Retrieves a fresh accessToken with the information in the [flow property]{@link GoogleAnalytics4OAuthSettings#flow } +- *Default:* Retrieves a fresh accessToken with the information in the [flow property]{@link GoogleBigQueryOAuthSettings#flow } -The access token to be used when interacting with Google Analytics 4. +The access token to be used when interacting with Google BigQuery. Note that if only the access token is provided AppFlow is not able to retrieve a fresh access token when the current one is expired --- -##### `endpoints`Optional +##### `endpoints`Optional ```typescript -public readonly endpoints: GoogleAnalytics4OAuthEndpoints; +public readonly endpoints: GoogleBigQueryOAuthEndpoints; ``` -- *Type:* GoogleAnalytics4OAuthEndpoints +- *Type:* GoogleBigQueryOAuthEndpoints The OAuth token and authorization endpoints. --- -##### `flow`Optional +##### `flow`Optional ```typescript -public readonly flow: GoogleAnalytics4OAuthFlow; +public readonly flow: GoogleBigQueryOAuthFlow; ``` -- *Type:* GoogleAnalytics4OAuthFlow +- *Type:* GoogleBigQueryOAuthFlow - *Default:* undefined. AppFlow will not request any new accessToken after expiry. The OAuth flow used for obtaining a new accessToken when the old is not present or expired. --- -### GoogleAnalytics4RefreshTokenGrantFlow +### GoogleBigQueryObject + +#### Initializer + +```typescript +import { GoogleBigQueryObject } from '@cdklabs/cdk-appflow' + +const googleBigQueryObject: GoogleBigQueryObject = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| dataset | string | *No description.* | +| project | string | *No description.* | +| table | string | *No description.* | + +--- + +##### `dataset`Required + +```typescript +public readonly dataset: string; +``` + +- *Type:* string + +--- + +##### `project`Required + +```typescript +public readonly project: string; +``` + +- *Type:* string + +--- + +##### `table`Required + +```typescript +public readonly table: string; +``` + +- *Type:* string + +--- + +### GoogleBigQueryRefreshTokenGrantFlow The OAuth elements required for the execution of the refresh token grant flow. -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4RefreshTokenGrantFlow } from '@cdklabs/cdk-appflow' +import { GoogleBigQueryRefreshTokenGrantFlow } from '@cdklabs/cdk-appflow' -const googleAnalytics4RefreshTokenGrantFlow: GoogleAnalytics4RefreshTokenGrantFlow = { ... } +const googleBigQueryRefreshTokenGrantFlow: GoogleBigQueryRefreshTokenGrantFlow = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| clientId | aws-cdk-lib.SecretValue | The id of the client app. | -| clientSecret | aws-cdk-lib.SecretValue | The secret of the client app. | -| refreshToken | aws-cdk-lib.SecretValue | A non-expired refresh token. | +| clientId | aws-cdk-lib.SecretValue | The id of the client app. | +| clientSecret | aws-cdk-lib.SecretValue | The secret of the client app. | +| refreshToken | aws-cdk-lib.SecretValue | A non-expired refresh token. | --- -##### `clientId`Optional +##### `clientId`Optional ```typescript public readonly clientId: SecretValue; @@ -7363,7 +7988,7 @@ The id of the client app. --- -##### `clientSecret`Optional +##### `clientSecret`Optional ```typescript public readonly clientSecret: SecretValue; @@ -7375,7 +8000,7 @@ The secret of the client app. --- -##### `refreshToken`Optional +##### `refreshToken`Optional ```typescript public readonly refreshToken: SecretValue; @@ -7387,29 +8012,29 @@ A non-expired refresh token. --- -### GoogleAnalytics4SourceProps +### GoogleBigQuerySourceProps -Properties of a Google Analytics v4 Source. +Properties of a Google BigQuery Source. -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4SourceProps } from '@cdklabs/cdk-appflow' +import { GoogleBigQuerySourceProps } from '@cdklabs/cdk-appflow' -const googleAnalytics4SourceProps: GoogleAnalytics4SourceProps = { ... } +const googleBigQuerySourceProps: GoogleBigQuerySourceProps = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| apiVersion | string | *No description.* | -| object | string | *No description.* | -| profile | GoogleAnalytics4ConnectorProfile | *No description.* | +| apiVersion | string | *No description.* | +| object | GoogleBigQueryObject | *No description.* | +| profile | GoogleBigQueryConnectorProfile | *No description.* | --- -##### `apiVersion`Required +##### `apiVersion`Required ```typescript public readonly apiVersion: string; @@ -7419,23 +8044,23 @@ public readonly apiVersion: string; --- -##### `object`Required +##### `object`Required ```typescript -public readonly object: string; +public readonly object: GoogleBigQueryObject; ``` -- *Type:* string +- *Type:* GoogleBigQueryObject --- -##### `profile`Required +##### `profile`Required ```typescript -public readonly profile: GoogleAnalytics4ConnectorProfile; +public readonly profile: GoogleBigQueryConnectorProfile; ``` -- *Type:* GoogleAnalytics4ConnectorProfile +- *Type:* GoogleBigQueryConnectorProfile --- @@ -12777,6 +13402,74 @@ The AppFlow type of the connector that this source is implemented for. --- +### GoogleBigQuerySource + +- *Implements:* ISource + +A class that represents a Google BigQuery Source. + +#### Initializers + +```typescript +import { GoogleBigQuerySource } from '@cdklabs/cdk-appflow' + +new GoogleBigQuerySource(props: GoogleBigQuerySourceProps) +``` + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| props | GoogleBigQuerySourceProps | *No description.* | + +--- + +##### `props`Required + +- *Type:* GoogleBigQuerySourceProps + +--- + +#### Methods + +| **Name** | **Description** | +| --- | --- | +| bind | *No description.* | + +--- + +##### `bind` + +```typescript +public bind(scope: IFlow): SourceFlowConfigProperty +``` + +###### `scope`Required + +- *Type:* IFlow + +--- + + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| connectorType | ConnectorType | The AppFlow type of the connector that this source is implemented for. | + +--- + +##### `connectorType`Required + +```typescript +public readonly connectorType: ConnectorType; +``` + +- *Type:* ConnectorType + +The AppFlow type of the connector that this source is implemented for. + +--- + + ### JdbcSmallDataScaleSource - *Implements:* ISource @@ -15184,7 +15877,7 @@ The AppFlow type of the connector that this source is implemented for. - *Extends:* aws-cdk-lib.IResource -- *Implemented By:* AmazonRdsForPostgreSqlConnectorProfile, ConnectorProfileBase, GoogleAnalytics4ConnectorProfile, JdbcSmallDataScaleConnectorProfile, MarketoConnectorProfile, MicrosoftDynamics365ConnectorProfile, MicrosoftSharepointOnlineConnectorProfile, RedshiftConnectorProfile, SAPOdataConnectorProfile, SalesforceConnectorProfile, SalesforceMarketingCloudConnectorProfile, ServiceNowConnectorProfile, SlackConnectorProfile, SnowflakeConnectorProfile, ZendeskConnectorProfile, IConnectorProfile +- *Implemented By:* AmazonRdsForPostgreSqlConnectorProfile, ConnectorProfileBase, GoogleAnalytics4ConnectorProfile, GoogleBigQueryConnectorProfile, JdbcSmallDataScaleConnectorProfile, MarketoConnectorProfile, MicrosoftDynamics365ConnectorProfile, MicrosoftSharepointOnlineConnectorProfile, RedshiftConnectorProfile, SAPOdataConnectorProfile, SalesforceConnectorProfile, SalesforceMarketingCloudConnectorProfile, ServiceNowConnectorProfile, SlackConnectorProfile, SnowflakeConnectorProfile, ZendeskConnectorProfile, IConnectorProfile #### Properties @@ -15596,7 +16289,7 @@ public bind(flow: IFlow, source: ISource): TaskProperty[] - *Extends:* IVertex -- *Implemented By:* GoogleAnalytics4Source, JdbcSmallDataScaleSource, MarketoSource, MicrosoftDynamics365Source, MicrosoftSharepointOnlineSource, S3Source, SAPOdataSource, SalesforceMarketingCloudSource, SalesforceSource, ServiceNowSource, SlackSource, ZendeskSource, ISource +- *Implemented By:* GoogleAnalytics4Source, GoogleBigQuerySource, JdbcSmallDataScaleSource, MarketoSource, MicrosoftDynamics365Source, MicrosoftSharepointOnlineSource, S3Source, SAPOdataSource, SalesforceMarketingCloudSource, SalesforceSource, ServiceNowSource, SlackSource, ZendeskSource, ISource A source of an AppFlow flow. @@ -15695,7 +16388,7 @@ A representation of a validation operation, that is an operation testing records ### IVertex -- *Implemented By:* AmazonRdsForPostgreSqlDestination, EventBridgeDestination, GoogleAnalytics4Source, JdbcSmallDataScaleSource, MarketoSource, MicrosoftDynamics365Source, MicrosoftSharepointOnlineSource, RedshiftDestination, S3Destination, S3Source, SAPOdataDestination, SAPOdataSource, SalesforceDestination, SalesforceMarketingCloudSource, SalesforceSource, ServiceNowSource, SlackSource, SnowflakeDestination, ZendeskSource, IDestination, ISource, IVertex +- *Implemented By:* AmazonRdsForPostgreSqlDestination, EventBridgeDestination, GoogleAnalytics4Source, GoogleBigQuerySource, JdbcSmallDataScaleSource, MarketoSource, MicrosoftDynamics365Source, MicrosoftSharepointOnlineSource, RedshiftDestination, S3Destination, S3Source, SAPOdataDestination, SAPOdataSource, SalesforceDestination, SalesforceMarketingCloudSource, SalesforceSource, ServiceNowSource, SlackSource, SnowflakeDestination, ZendeskSource, IDestination, ISource, IVertex An interface representing a vertex, i.e. a source or a destination of an AppFlow flow. @@ -15860,6 +16553,21 @@ The AppFlow type of the connector that this source is implemented for. --- +### GoogleBigQueryApiVersion + +#### Members + +| **Name** | **Description** | +| --- | --- | +| V2 | *No description.* | + +--- + +##### `V2` + +--- + + ### JdbcDriver #### Members diff --git a/package.json b/package.json index 3f5c223b..1c8a0fb4 100644 --- a/package.json +++ b/package.json @@ -112,8 +112,8 @@ "organization": true }, "devDependencies": { - "@aws-cdk/aws-glue-alpha": "2.116.1-alpha.0", - "@aws-cdk/aws-redshift-alpha": "2.116.1-alpha.0", + "@aws-cdk/aws-glue-alpha": "2.121.1-alpha.0", + "@aws-cdk/aws-redshift-alpha": "2.121.1-alpha.0", "@aws-cdk/integ-runner": "latest", "@aws-cdk/integ-tests-alpha": "latest", "@types/jest": "^27", @@ -121,8 +121,8 @@ "@typescript-eslint/eslint-plugin": "^6", "@typescript-eslint/parser": "^6", "aws-cdk": "^2", - "aws-cdk-lib": "2.116.1", - "cdklabs-projen-project-types": "^0.1.184", + "aws-cdk-lib": "2.121.1", + "cdklabs-projen-project-types": "^0.1.186", "constructs": "10.0.5", "esbuild": "^0.19.11", "eslint": "^8", @@ -135,16 +135,16 @@ "jsii-docgen": "^8.0.56", "jsii-pacmak": "^1.94.0", "jsii-rosetta": "^5.3.3", - "projen": "^0.77.6", + "projen": "^0.79.1", "standard-version": "^9", "ts-jest": "^27", "ts-node": "^10.9.2", "typescript": "^5.2.2" }, "peerDependencies": { - "@aws-cdk/aws-glue-alpha": "2.116.1-alpha.0", - "@aws-cdk/aws-redshift-alpha": "2.116.1-alpha.0", - "aws-cdk-lib": "^2.116.1", + "@aws-cdk/aws-glue-alpha": "2.121.1-alpha.0", + "@aws-cdk/aws-redshift-alpha": "2.121.1-alpha.0", + "aws-cdk-lib": "^2.121.1", "constructs": "^10.0.5" }, "dependencies": { diff --git a/src/googlebigquery/index.ts b/src/googlebigquery/index.ts new file mode 100644 index 00000000..bba9c6ea --- /dev/null +++ b/src/googlebigquery/index.ts @@ -0,0 +1,8 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +export * from './type'; +export * from './profile'; +export * from './source'; +export * from './util'; \ No newline at end of file diff --git a/src/googlebigquery/profile.ts b/src/googlebigquery/profile.ts new file mode 100644 index 00000000..80309a49 --- /dev/null +++ b/src/googlebigquery/profile.ts @@ -0,0 +1,137 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { SecretValue } from 'aws-cdk-lib'; +import { CfnConnectorProfile } from 'aws-cdk-lib/aws-appflow'; +import { Construct } from 'constructs'; +import { GoogleBigQueryConnectorType } from './type'; +import { ConnectorAuthenticationType } from '../core/connectors/connector-authentication-type'; +import { ConnectorProfileBase, ConnectorProfileProps } from '../core/connectors/connector-profile'; +import { OAuth2GrantType as OAuthGrantType } from '../core/connectors/oauth2-granttype'; + +export interface GoogleBigQueryConnectorProfileProps extends ConnectorProfileProps { + readonly oAuth: GoogleBigQueryOAuthSettings; +} + +/** + * Google's OAuth token and authorization endpoints + */ +export interface GoogleBigQueryOAuthEndpoints { + /** + * The OAuth token endpoint URI + */ + readonly token?: string; + + /** + * The OAuth authorization endpoint URI + */ + readonly authorization?: string; +} + +/** + * The OAuth elements required for the execution of the refresh token grant flow. + */ +export interface GoogleBigQueryRefreshTokenGrantFlow { + /** + * A non-expired refresh token. + */ + readonly refreshToken?: SecretValue; + + /** + * The secret of the client app. + */ + readonly clientSecret?: SecretValue; + + /** + * The id of the client app. + */ + readonly clientId?: SecretValue; +} + +/** + * Represents the OAuth flow enabled for the GA4 + */ +export interface GoogleBigQueryOAuthFlow { + /** + * The details required for executing the refresh token grant flow + */ + readonly refreshTokenGrant: GoogleBigQueryRefreshTokenGrantFlow; +} + +export interface GoogleBigQueryOAuthSettings { + + /** + * The access token to be used when interacting with Google BigQuery + * + * Note that if only the access token is provided AppFlow is not able to retrieve a fresh access token when the current one is expired + * + * @default Retrieves a fresh accessToken with the information in the [flow property]{@link GoogleBigQueryOAuthSettings#flow} + */ + readonly accessToken?: SecretValue; + + /** + * The OAuth flow used for obtaining a new accessToken when the old is not present or expired. + * + * @default undefined. AppFlow will not request any new accessToken after expiry. + */ + readonly flow?: GoogleBigQueryOAuthFlow; + + /** + * The OAuth token and authorization endpoints. + */ + readonly endpoints?: GoogleBigQueryOAuthEndpoints; +} + +export class GoogleBigQueryConnectorProfile extends ConnectorProfileBase { + + public static fromConnectionProfileArn(scope: Construct, id: string, arn: string) { + return this._fromConnectorProfileAttributes(scope, id, { arn }) as GoogleBigQueryConnectorProfile; + } + + public static fromConnectionProfileName(scope: Construct, id: string, name: string) { + return this._fromConnectorProfileAttributes(scope, id, { name }) as GoogleBigQueryConnectorProfile; + } + + private static readonly defaultTokenEndpoint: string = 'https://oauth2.googleapis.com/token'; + + constructor(scope: Construct, id: string, props: GoogleBigQueryConnectorProfileProps) { + super(scope, id, props, GoogleBigQueryConnectorType.instance); + } + + protected buildConnectorProfileProperties( + props: ConnectorProfileProps, + ): CfnConnectorProfile.ConnectorProfilePropertiesProperty { + const properties = (props as GoogleBigQueryConnectorProfileProps); + return { + customConnector: { + oAuth2Properties: { + // INFO: even if we're using a refresh token grant flow this property is required + oAuth2GrantType: OAuthGrantType.AUTHORIZATION_CODE, + // INFO: even if we provide only the access token this property is required + // TODO: think about if this is correct. token can be IResolvable + tokenUrl: properties.oAuth.endpoints?.token ?? GoogleBigQueryConnectorProfile.defaultTokenEndpoint, + }, + }, + }; + } + + protected buildConnectorProfileCredentials( + props: ConnectorProfileProps, + ): CfnConnectorProfile.ConnectorProfileCredentialsProperty { + const properties = (props as GoogleBigQueryConnectorProfileProps); + + return { + customConnector: { + oauth2: { + // INFO: when using Refresh Token Grant Flow - access token property is required + accessToken: properties.oAuth.accessToken?.unsafeUnwrap() ?? 'dummyAccessToken', + refreshToken: properties.oAuth.flow?.refreshTokenGrant.refreshToken?.unsafeUnwrap(), + clientId: properties.oAuth.flow?.refreshTokenGrant.clientId?.unsafeUnwrap(), + clientSecret: properties.oAuth.flow?.refreshTokenGrant.clientSecret?.unsafeUnwrap(), + }, + authenticationType: ConnectorAuthenticationType.OAUTH2, + }, + }; + } +} \ No newline at end of file diff --git a/src/googlebigquery/source.ts b/src/googlebigquery/source.ts new file mode 100644 index 00000000..3e89d07c --- /dev/null +++ b/src/googlebigquery/source.ts @@ -0,0 +1,65 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { CfnFlow } from 'aws-cdk-lib/aws-appflow'; +import { IConstruct } from 'constructs'; +import { GoogleBigQueryConnectorProfile } from './profile'; +import { GoogleBigQueryConnectorType } from './type'; +import { ConnectorType } from '../core/connectors/connector-type'; +import { IFlow } from '../core/flows'; +import { ISource } from '../core/vertices'; + +export interface GoogleBigQueryObject { + readonly project: string; + readonly dataset: string; + readonly table: string; +} + +/** + * Properties of a Google BigQuery Source + */ +export interface GoogleBigQuerySourceProps { + readonly profile: GoogleBigQueryConnectorProfile; + readonly apiVersion: string; + readonly object: GoogleBigQueryObject; +} + +/** + * A class that represents a Google BigQuery Source + */ +export class GoogleBigQuerySource implements ISource { + + /** + * The AppFlow type of the connector that this source is implemented for + */ + public readonly connectorType: ConnectorType = GoogleBigQueryConnectorType.instance; + + constructor(private readonly props: GoogleBigQuerySourceProps) { } + + bind(scope: IFlow): CfnFlow.SourceFlowConfigProperty { + + this.tryAddNodeDependency(scope, this.props.profile); + + return { + connectorType: this.connectorType.asProfileConnectorType, + connectorProfileName: this.props.profile.name, + apiVersion: this.props.apiVersion, + sourceConnectorProperties: this.buildSourceConnectorProperties(), + }; + } + + private buildSourceConnectorProperties(): CfnFlow.SourceConnectorPropertiesProperty { + return { + customConnector: { + entityName: `table/${this.props.object.project}/${this.props.object.dataset}/${this.props.object.table}`, + }, + }; + } + + private tryAddNodeDependency(scope: IConstruct, resource?: IConstruct | string): void { + if (resource && typeof resource !== 'string') { + scope.node.addDependency(resource); + } + } +} \ No newline at end of file diff --git a/src/googlebigquery/type.ts b/src/googlebigquery/type.ts new file mode 100644 index 00000000..55cb172d --- /dev/null +++ b/src/googlebigquery/type.ts @@ -0,0 +1,24 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { ConnectorType } from '../core/connectors/connector-type'; + +/** + * @internal + */ +export class GoogleBigQueryConnectorType extends ConnectorType { + + public static get instance(): ConnectorType { + if (!GoogleBigQueryConnectorType.actualInstance) { + GoogleBigQueryConnectorType.actualInstance = new GoogleBigQueryConnectorType(); + } + return GoogleBigQueryConnectorType.actualInstance; + } + + private static actualInstance: ConnectorType; + + constructor() { + super('Google BigQuery', true); + } +} \ No newline at end of file diff --git a/src/googlebigquery/util.ts b/src/googlebigquery/util.ts new file mode 100644 index 00000000..e2672db3 --- /dev/null +++ b/src/googlebigquery/util.ts @@ -0,0 +1,7 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +export enum GoogleBigQueryApiVersion { + V2 = 'v2' +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index fc225883..d66baad3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ export * from './core'; export * from './amazonrdsforpostgresql'; export * from './eventbridge'; export * from './googleanalytics4'; +export * from './googlebigquery'; export * from './jdbcsmalldatascale'; export * from './marketo'; export * from './microsoftdynamics365'; diff --git a/test/googlebigquery/profile.test.ts b/test/googlebigquery/profile.test.ts new file mode 100644 index 00000000..95efeb7d --- /dev/null +++ b/test/googlebigquery/profile.test.ts @@ -0,0 +1,286 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { SecretValue, Stack } from 'aws-cdk-lib'; +import { Template } from 'aws-cdk-lib/assertions'; +import { Key } from 'aws-cdk-lib/aws-kms'; +import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; + +import { + GoogleBigQueryConnectorProfile, +} from '../../src'; + +describe('GoogleBigQueryConnectorProfile', () => { + + test('OAuth2 profile with direct client credentials exists in the stack', () => { + const stack = new Stack(undefined, 'TestStack', { env: { account: '12345678', region: 'dummy' } }); + + new GoogleBigQueryConnectorProfile(stack, 'TestProfile', { + oAuth: { + accessToken: SecretValue.unsafePlainText('accessToken'), + flow: { + refreshTokenGrant: { + refreshToken: SecretValue.unsafePlainText('refreshToken'), + clientId: SecretValue.unsafePlainText('clientId'), + clientSecret: SecretValue.unsafePlainText('clientSecret'), + }, + }, + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::AppFlow::ConnectorProfile', { + ConnectionMode: 'Public', + ConnectorProfileName: 'TestProfile', + ConnectorType: 'CustomConnector', + ConnectorLabel: 'Google BigQuery', + ConnectorProfileConfig: { + ConnectorProfileCredentials: { + CustomConnector: { + AuthenticationType: 'OAUTH2', + Oauth2: { + AccessToken: 'accessToken', + ClientId: 'clientId', + ClientSecret: 'clientSecret', + RefreshToken: 'refreshToken', + }, + }, + }, + ConnectorProfileProperties: { + CustomConnector: { + OAuth2Properties: { + OAuth2GrantType: 'AUTHORIZATION_CODE', + TokenUrl: 'https://oauth2.googleapis.com/token', + }, + }, + }, + }, + }); + }); + + test('OAuth2 profile with client credentials as secret elements exists in the stack', () => { + const stack = new Stack(undefined, 'TestStack', { env: { account: '12345678', region: 'dummy' } }); + + const secret = new Secret(stack, 'TestSecret'); + + new GoogleBigQueryConnectorProfile(stack, 'TestProfile', { + oAuth: { + accessToken: secret.secretValueFromJson('accessToken'), + flow: { + refreshTokenGrant: { + refreshToken: secret.secretValueFromJson('refreshToken'), + clientId: secret.secretValueFromJson('clientId'), + clientSecret: secret.secretValueFromJson('clientSecret'), + }, + }, + endpoints: { + token: secret.secretValueFromJson('tokenUrl').toString(), + }, + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::AppFlow::ConnectorProfile', { + ConnectionMode: 'Public', + ConnectorProfileName: 'TestProfile', + ConnectorType: 'CustomConnector', + ConnectorLabel: 'Google BigQuery', + ConnectorProfileConfig: { + ConnectorProfileCredentials: { + CustomConnector: { + AuthenticationType: 'OAUTH2', + Oauth2: { + AccessToken: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:accessToken::}}', + ], + ], + }, + ClientId: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:clientId::}}', + ], + ], + }, + ClientSecret: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:clientSecret::}}', + ], + ], + }, + RefreshToken: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:refreshToken::}}', + ], + ], + }, + }, + }, + }, + ConnectorProfileProperties: { + CustomConnector: { + OAuth2Properties: { + OAuth2GrantType: 'AUTHORIZATION_CODE', + TokenUrl: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:tokenUrl::}}', + ], + ], + }, + }, + }, + }, + }, + }); + }); + + + test('OAuth2 profile with a dedicated KMS key and client credentials as secret elements exists in the stack', () => { + const stack = new Stack(undefined, 'TestStack', { env: { account: '12345678', region: 'dummy' } }); + + const key = new Key(stack, 'TestKey'); + + const secret = new Secret(stack, 'TestSecret'); + + new GoogleBigQueryConnectorProfile(stack, 'TestProfile', { + key: key, + oAuth: { + accessToken: secret.secretValueFromJson('accessToken'), + flow: { + refreshTokenGrant: { + refreshToken: secret.secretValueFromJson('refreshToken'), + clientId: secret.secretValueFromJson('clientId'), + clientSecret: secret.secretValueFromJson('clientSecret'), + }, + }, + endpoints: { + token: secret.secretValueFromJson('tokenUrl').toString(), + }, + }, + }); + + Template.fromStack(stack).hasResource('AWS::AppFlow::ConnectorProfile', { + Properties: { + ConnectionMode: 'Public', + ConnectorProfileName: 'TestProfile', + ConnectorType: 'CustomConnector', + ConnectorLabel: 'Google BigQuery', + ConnectorProfileConfig: { + ConnectorProfileCredentials: { + CustomConnector: { + AuthenticationType: 'OAUTH2', + Oauth2: { + AccessToken: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:accessToken::}}', + ], + ], + }, + ClientId: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:clientId::}}', + ], + ], + }, + ClientSecret: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:clientSecret::}}', + ], + ], + }, + RefreshToken: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:refreshToken::}}', + ], + ], + }, + }, + }, + }, + ConnectorProfileProperties: { + CustomConnector: { + OAuth2Properties: { + OAuth2GrantType: 'AUTHORIZATION_CODE', + TokenUrl: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:tokenUrl::}}', + ], + ], + }, + }, + }, + }, + }, + KMSArn: { + 'Fn::GetAtt': [ + 'TestKey4CACAF33', + 'Arn', + ], + }, + }, + DependsOn: [ + 'TestKey4CACAF33', + ], + }); + }); + +}); \ No newline at end of file diff --git a/test/googlebigquery/source.test.ts b/test/googlebigquery/source.test.ts new file mode 100644 index 00000000..779bdfee --- /dev/null +++ b/test/googlebigquery/source.test.ts @@ -0,0 +1,224 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { SecretValue, Stack } from 'aws-cdk-lib'; +import { Template } from 'aws-cdk-lib/assertions'; +import { Bucket } from 'aws-cdk-lib/aws-s3'; + +import { + Mapping, + GoogleBigQueryApiVersion, + GoogleBigQueryConnectorProfile, + GoogleBigQueryConnectorType, + GoogleBigQuerySource, + OnDemandFlow, + S3Destination, +} from '../../src'; + +describe('GoogleBigQuerySource', () => { + + test('Source with only connector name', () => { + const stack = new Stack(undefined, 'TestStack'); + const source = new GoogleBigQuerySource({ + profile: GoogleBigQueryConnectorProfile.fromConnectionProfileName(stack, 'TestProfile', 'dummy-profile'), + object: { + project: 'projectName', + dataset: 'datasetName', + table: 'tableName', + }, + apiVersion: GoogleBigQueryApiVersion.V2, + }); + + const expectedConnectorType = GoogleBigQueryConnectorType.instance; + expect(source.connectorType.asProfileConnectorLabel).toEqual(expectedConnectorType.asProfileConnectorLabel); + expect(source.connectorType.asProfileConnectorType).toEqual(expectedConnectorType.asProfileConnectorType); + expect(source.connectorType.asTaskConnectorOperatorOrigin).toEqual(expectedConnectorType.asTaskConnectorOperatorOrigin); + expect(source.connectorType.isCustom).toEqual(expectedConnectorType.isCustom); + }); + + test('Source in a Flow is in the stack', () => { + const stack = new Stack(undefined, 'TestStack'); + const source = new GoogleBigQuerySource({ + profile: GoogleBigQueryConnectorProfile.fromConnectionProfileName(stack, 'TestProfile', 'dummy-profile'), + object: { + project: 'projectName', + dataset: 'datasetName', + table: 'tableName', + }, + apiVersion: GoogleBigQueryApiVersion.V2, + }); + + const destination = new S3Destination({ + location: { bucket: new Bucket(stack, 'TestBucket') }, + }); + + new OnDemandFlow(stack, 'TestFlow', { + source: source, + destination: destination, + mappings: [Mapping.mapAll()], + }); + + Template.fromStack(stack).hasResourceProperties('AWS::AppFlow::Flow', { + DestinationFlowConfigList: [ + { + ConnectorType: 'S3', + DestinationConnectorProperties: { + S3: { + BucketName: { + Ref: 'TestBucket560B80BC', + }, + }, + }, + }, + ], + FlowName: 'TestFlow', + SourceFlowConfig: { + ApiVersion: 'v2', + ConnectorProfileName: 'dummy-profile', + ConnectorType: 'CustomConnector', + SourceConnectorProperties: { + CustomConnector: { + EntityName: 'table/projectName/datasetName/tableName', + }, + }, + }, + Tasks: [ + { + ConnectorOperator: { + CustomConnector: 'NO_OP', + }, + SourceFields: [], + TaskProperties: [ + { + Key: 'EXCLUDE_SOURCE_FIELDS_LIST', + Value: '[]', + }, + ], + TaskType: 'Map_all', + }, + ], + TriggerConfig: { + TriggerType: 'OnDemand', + }, + }); + }); + + test('Source for dummy-profile in a Flow is in the stack', () => { + const stack = new Stack(undefined, 'TestStack'); + + const profile = new GoogleBigQueryConnectorProfile(stack, 'TestProfile', { + oAuth: { + accessToken: SecretValue.unsafePlainText('accessToken'), + flow: { + refreshTokenGrant: { + refreshToken: SecretValue.unsafePlainText('refreshToken'), + clientId: SecretValue.unsafePlainText('clientId'), + clientSecret: SecretValue.unsafePlainText('clientSecret'), + }, + }, + }, + }); + + const source = new GoogleBigQuerySource({ + profile: profile, + object: { + project: 'projectName', + dataset: 'datasetName', + table: 'tableName', + }, + apiVersion: GoogleBigQueryApiVersion.V2, + }); + + const destination = new S3Destination({ + location: { bucket: new Bucket(stack, 'TestBucket') }, + }); + + new OnDemandFlow(stack, 'TestFlow', { + source: source, + destination: destination, + mappings: [Mapping.mapAll()], + }); + + const template = Template.fromStack(stack); + + template.hasResourceProperties('AWS::AppFlow::ConnectorProfile', { + ConnectionMode: 'Public', + ConnectorProfileName: 'TestProfile', + ConnectorType: 'CustomConnector', + ConnectorLabel: 'Google BigQuery', + ConnectorProfileConfig: { + ConnectorProfileCredentials: { + CustomConnector: { + AuthenticationType: 'OAUTH2', + Oauth2: { + AccessToken: 'accessToken', + ClientId: 'clientId', + ClientSecret: 'clientSecret', + RefreshToken: 'refreshToken', + }, + }, + }, + ConnectorProfileProperties: { + CustomConnector: { + OAuth2Properties: { + OAuth2GrantType: 'AUTHORIZATION_CODE', + TokenUrl: 'https://oauth2.googleapis.com/token', + }, + }, + }, + }, + }); + + template.hasResource('AWS::AppFlow::Flow', { + Properties: { + DestinationFlowConfigList: [ + { + ConnectorType: 'S3', + DestinationConnectorProperties: { + S3: { + BucketName: { + Ref: 'TestBucket560B80BC', + }, + }, + }, + }, + ], + FlowName: 'TestFlow', + SourceFlowConfig: { + ApiVersion: 'v2', + ConnectorProfileName: 'TestProfile', + ConnectorType: 'CustomConnector', + SourceConnectorProperties: { + CustomConnector: { + EntityName: 'table/projectName/datasetName/tableName', + }, + }, + }, + Tasks: [ + { + ConnectorOperator: { + CustomConnector: 'NO_OP', + }, + SourceFields: [], + TaskProperties: [ + { + Key: 'EXCLUDE_SOURCE_FIELDS_LIST', + Value: '[]', + }, + ], + TaskType: 'Map_all', + }, + ], + TriggerConfig: { + TriggerType: 'OnDemand', + }, + }, + DependsOn: [ + 'TestBucketPolicyBA12ED38', + 'TestBucket560B80BC', + 'TestProfile45C36389', + ], + }); + }); +}); diff --git a/test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/TestStack.assets.json deleted file mode 100644 index c596202b..00000000 --- a/test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/TestStack.assets.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "version": "32.0.0", - "files": { - "57ce1be8e51f8f48dd339fadf0f4391c0dc7b4e9e109e23327a9c2a959428eaf": { - "source": { - "path": "TestStack.template.json", - "packaging": "file" - }, - "destinations": { - "current_account-current_region": { - "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "57ce1be8e51f8f48dd339fadf0f4391c0dc7b4e9e109e23327a9c2a959428eaf.json", - "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" - } - } - } - }, - "dockerImages": {} -} \ No newline at end of file diff --git a/test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/TestStack.template.json b/test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/TestStack.template.json deleted file mode 100644 index 40fc88d1..00000000 --- a/test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/TestStack.template.json +++ /dev/null @@ -1,165 +0,0 @@ -{ - "Resources": { - "TestConnectorProfileD7470107": { - "Type": "AWS::AppFlow::ConnectorProfile", - "Properties": { - "ConnectionMode": "Public", - "ConnectorProfileName": "TestConnectorProfile", - "ConnectorType": "CustomConnector", - "ConnectorLabel": "AmazonRDSPostgreSQL", - "ConnectorProfileConfig": { - "ConnectorProfileCredentials": { - "CustomConnector": { - "AuthenticationType": "CUSTOM", - "Custom": { - "CredentialsMap": { - "username": { - "Fn::Join": [ - "", - [ - "{{resolve:secretsmanager:arn:", - { - "Ref": "AWS::Partition" - }, - ":secretsmanager:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":secret:appflow/rdspostgres:SecretString:username::}}" - ] - ] - }, - "password": { - "Fn::Join": [ - "", - [ - "{{resolve:secretsmanager:arn:", - { - "Ref": "AWS::Partition" - }, - ":secretsmanager:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":secret:appflow/rdspostgres:SecretString:password::}}" - ] - ] - }, - "driver": "postgresql", - "hostname": { - "Fn::Join": [ - "", - [ - "{{resolve:secretsmanager:arn:", - { - "Ref": "AWS::Partition" - }, - ":secretsmanager:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":secret:appflow/rdspostgres:SecretString:hostname::}}" - ] - ] - }, - "port": { - "Fn::Join": [ - "", - [ - "{{resolve:secretsmanager:arn:", - { - "Ref": "AWS::Partition" - }, - ":secretsmanager:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":secret:appflow/rdspostgres:SecretString:port::}}" - ] - ] - }, - "database": { - "Fn::Join": [ - "", - [ - "{{resolve:secretsmanager:arn:", - { - "Ref": "AWS::Partition" - }, - ":secretsmanager:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":secret:appflow/rdspostgres:SecretString:database::}}" - ] - ] - } - }, - "CustomAuthenticationType": "CUSTOM" - } - } - }, - "ConnectorProfileProperties": { - "CustomConnector": { - "ProfileProperties": {} - } - } - } - } - } - }, - "Parameters": { - "BootstrapVersion": { - "Type": "AWS::SSM::Parameter::Value", - "Default": "/cdk-bootstrap/hnb659fds/version", - "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" - } - }, - "Rules": { - "CheckBootstrapVersion": { - "Assertions": [ - { - "Assert": { - "Fn::Not": [ - { - "Fn::Contains": [ - [ - "1", - "2", - "3", - "4", - "5" - ], - { - "Ref": "BootstrapVersion" - } - ] - } - ] - }, - "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." - } - ] - } - } -} \ No newline at end of file diff --git a/test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/cdk.out b/test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/cdk.out deleted file mode 100644 index f0b901e7..00000000 --- a/test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/cdk.out +++ /dev/null @@ -1 +0,0 @@ -{"version":"32.0.0"} \ No newline at end of file diff --git a/test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/TestStack.assets.json index 0e94becc..e8760ee8 100644 --- a/test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/TestStack.assets.json +++ b/test/integ/ondemand-googleanalytics4-to-s3.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { diff --git a/test/integ/ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot/TestStack.assets.json index 6d9faff5..48f55eff 100644 --- a/test/integ/ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot/TestStack.assets.json +++ b/test/integ/ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "6861d545ce03a23ba70eed7607eb9d7591a85a706ff66d03d5aad22687d28078": { "source": { diff --git a/test/integ/ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot/cdk.out b/test/integ/ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot/cdk.out new file mode 100644 index 00000000..1f0068d3 --- /dev/null +++ b/test/integ/ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot/cdk.out @@ -0,0 +1 @@ +{"version":"36.0.0"} \ No newline at end of file diff --git a/test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/manifest.json b/test/integ/ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot/manifest.json similarity index 72% rename from test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/manifest.json rename to test/integ/ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot/manifest.json index 658329c1..98d81131 100644 --- a/test/integ/ondemand-amazonrdsforpostgresql-to-s3.integ.snapshot/manifest.json +++ b/test/integ/ondemand-jdbcsmalldatascale-to-amazonrdsforpostgresql.integ.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "artifacts": { "TestStack.assets": { "type": "cdk:asset-manifest", @@ -14,10 +14,11 @@ "environment": "aws://unknown-account/unknown-region", "properties": { "templateFile": "TestStack.template.json", + "terminationProtection": false, "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/57ce1be8e51f8f48dd339fadf0f4391c0dc7b4e9e109e23327a9c2a959428eaf.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/6861d545ce03a23ba70eed7607eb9d7591a85a706ff66d03d5aad22687d28078.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -33,10 +34,22 @@ "TestStack.assets" ], "metadata": { - "/TestStack/TestConnectorProfile/TestConnectorProfile": [ + "/TestStack/JdbcSmallTestConnectorProfile/JdbcSmallTestConnectorProfile": [ { "type": "aws:cdk:logicalId", - "data": "TestConnectorProfileD7470107" + "data": "JdbcSmallTestConnectorProfileD137BC4A" + } + ], + "/TestStack/AmazonRdsTestConnectorProfile/AmazonRdsTestConnectorProfile": [ + { + "type": "aws:cdk:logicalId", + "data": "AmazonRdsTestConnectorProfileDA7BF797" + } + ], + "/TestStack/OnDemandFlow/OnDemandFlow": [ + { + "type": "aws:cdk:logicalId", + "data": "OnDemandFlow4ECA54C5" } ], "/TestStack/BootstrapVersion": [ diff --git a/test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/TestStack.assets.json index 2959cbfa..764b8c7d 100644 --- a/test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/TestStack.assets.json +++ b/test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { diff --git a/test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/cdk.out b/test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/cdk.out new file mode 100644 index 00000000..1f0068d3 --- /dev/null +++ b/test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/cdk.out @@ -0,0 +1 @@ +{"version":"36.0.0"} \ No newline at end of file diff --git a/test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/manifest.json b/test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/manifest.json new file mode 100644 index 00000000..0ee3782f --- /dev/null +++ b/test/integ/ondemand-jdbcsmalldatascale-to-s3.integ.snapshot/manifest.json @@ -0,0 +1,95 @@ +{ + "version": "36.0.0", + "artifacts": { + "TestStack.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "TestStack.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "TestStack": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "TestStack.template.json", + "terminationProtection": false, + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/85e6c489b6ba6b249b35b568479083f37b5159ea1c2713c1812dfda7f164f585.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "TestStack.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "TestStack.assets" + ], + "metadata": { + "/TestStack/TestConnectorProfile/TestConnectorProfile": [ + { + "type": "aws:cdk:logicalId", + "data": "TestConnectorProfileD7470107" + } + ], + "/TestStack/TestBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "TestBucket560B80BC" + } + ], + "/TestStack/TestBucket/Policy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "TestBucketPolicyBA12ED38" + } + ], + "/TestStack/TestBucket/AutoDeleteObjectsCustomResource/Default": [ + { + "type": "aws:cdk:logicalId", + "data": "TestBucketAutoDeleteObjectsCustomResource8FEAABD5" + } + ], + "/TestStack/Custom::S3AutoDeleteObjectsCustomResourceProvider/Role": [ + { + "type": "aws:cdk:logicalId", + "data": "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" + } + ], + "/TestStack/Custom::S3AutoDeleteObjectsCustomResourceProvider/Handler": [ + { + "type": "aws:cdk:logicalId", + "data": "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F" + } + ], + "/TestStack/OnDemandFlow/OnDemandFlow": [ + { + "type": "aws:cdk:logicalId", + "data": "OnDemandFlow4ECA54C5" + } + ], + "/TestStack/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/TestStack/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "TestStack" + } + } +} \ No newline at end of file diff --git a/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/TestStack.assets.json index 2ff3d4b8..bb96311c 100644 --- a/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/TestStack.assets.json +++ b/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { diff --git a/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/asset.b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6/index.js b/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/asset.b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6/index.js new file mode 100644 index 00000000..ac2119e0 --- /dev/null +++ b/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/asset.b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6/index.js @@ -0,0 +1 @@ +"use strict";var C=Object.create,i=Object.defineProperty,I=Object.getOwnPropertyDescriptor,w=Object.getOwnPropertyNames,P=Object.getPrototypeOf,A=Object.prototype.hasOwnProperty,L=(e,t)=>{for(var o in t)i(e,o,{get:t[o],enumerable:!0})},d=(e,t,o,r)=>{if(t&&typeof t=="object"||typeof t=="function")for(let s of w(t))!A.call(e,s)&&s!==o&&i(e,s,{get:()=>t[s],enumerable:!(r=I(t,s))||r.enumerable});return e},l=(e,t,o)=>(o=e!=null?C(P(e)):{},d(t||!e||!e.__esModule?i(o,"default",{value:e,enumerable:!0}):o,e)),k=e=>d(i({},"__esModule",{value:!0}),e),U={};L(U,{autoDeleteHandler:()=>S,handler:()=>_}),module.exports=k(U);var h=require("@aws-sdk/client-s3"),y=l(require("https")),m=l(require("url")),a={sendHttpRequest:T,log:b,includeStackTraces:!0,userHandlerIndex:"./index"},p="AWSCDK::CustomResourceProviderFramework::CREATE_FAILED",B="AWSCDK::CustomResourceProviderFramework::MISSING_PHYSICAL_ID";function R(e){return async(t,o)=>{let r={...t,ResponseURL:"..."};if(a.log(JSON.stringify(r,void 0,2)),t.RequestType==="Delete"&&t.PhysicalResourceId===p){a.log("ignoring DELETE event caused by a failed CREATE event"),await u("SUCCESS",t);return}try{let s=await e(r,o),n=D(t,s);await u("SUCCESS",n)}catch(s){let n={...t,Reason:a.includeStackTraces?s.stack:s.message};n.PhysicalResourceId||(t.RequestType==="Create"?(a.log("CREATE failed, responding with a marker physical resource id so that the subsequent DELETE will be ignored"),n.PhysicalResourceId=p):a.log(`ERROR: Malformed event. "PhysicalResourceId" is required: ${JSON.stringify(t)}`)),await u("FAILED",n)}}}function D(e,t={}){let o=t.PhysicalResourceId??e.PhysicalResourceId??e.RequestId;if(e.RequestType==="Delete"&&o!==e.PhysicalResourceId)throw new Error(`DELETE: cannot change the physical resource ID from "${e.PhysicalResourceId}" to "${t.PhysicalResourceId}" during deletion`);return{...e,...t,PhysicalResourceId:o}}async function u(e,t){let o={Status:e,Reason:t.Reason??e,StackId:t.StackId,RequestId:t.RequestId,PhysicalResourceId:t.PhysicalResourceId||B,LogicalResourceId:t.LogicalResourceId,NoEcho:t.NoEcho,Data:t.Data};a.log("submit response to cloudformation",o);let r=JSON.stringify(o),s=m.parse(t.ResponseURL),n={hostname:s.hostname,path:s.path,method:"PUT",headers:{"content-type":"","content-length":Buffer.byteLength(r,"utf8")}};await O({attempts:5,sleep:1e3},a.sendHttpRequest)(n,r)}async function T(e,t){return new Promise((o,r)=>{try{let s=y.request(e,n=>o());s.on("error",r),s.write(t),s.end()}catch(s){r(s)}})}function b(e,...t){console.log(e,...t)}function O(e,t){return async(...o)=>{let r=e.attempts,s=e.sleep;for(;;)try{return await t(...o)}catch(n){if(r--<=0)throw n;await x(Math.floor(Math.random()*s)),s*=2}}}async function x(e){return new Promise(t=>setTimeout(t,e))}var g="aws-cdk:auto-delete-objects",H=JSON.stringify({Version:"2012-10-17",Statement:[]}),c=new h.S3({}),_=R(S);async function S(e){switch(e.RequestType){case"Create":return;case"Update":return F(e);case"Delete":return f(e.ResourceProperties?.BucketName)}}async function F(e){let t=e,o=t.OldResourceProperties?.BucketName,r=t.ResourceProperties?.BucketName;if(r!=null&&o!=null&&r!==o)return f(o)}async function N(e){try{let t=(await c.getBucketPolicy({Bucket:e}))?.Policy??H,o=JSON.parse(t);o.Statement.push({Principal:"*",Effect:"Deny",Action:["s3:PutObject"],Resource:[`arn:aws:s3:::${e}/*`]}),await c.putBucketPolicy({Bucket:e,Policy:JSON.stringify(o)})}catch(t){if(t.name==="NoSuchBucket")throw t;console.log(`Could not set new object deny policy on bucket '${e}' prior to deletion.`)}}async function E(e){let t=await c.listObjectVersions({Bucket:e}),o=[...t.Versions??[],...t.DeleteMarkers??[]];if(o.length===0)return;let r=o.map(s=>({Key:s.Key,VersionId:s.VersionId}));await c.deleteObjects({Bucket:e,Delete:{Objects:r}}),t?.IsTruncated&&await E(e)}async function f(e){if(!e)throw new Error("No BucketName was provided.");try{if(!await W(e)){console.log(`Bucket does not have '${g}' tag, skipping cleaning.`);return}await N(e),await E(e)}catch(t){if(t.name==="NoSuchBucket"){console.log(`Bucket '${e}' does not exist.`);return}throw t}}async function W(e){return(await c.getBucketTagging({Bucket:e})).TagSet?.some(o=>o.Key===g&&o.Value==="true")} diff --git a/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/cdk.out b/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/cdk.out new file mode 100644 index 00000000..1f0068d3 --- /dev/null +++ b/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/cdk.out @@ -0,0 +1 @@ +{"version":"36.0.0"} \ No newline at end of file diff --git a/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/manifest.json b/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/manifest.json new file mode 100644 index 00000000..29df2dab --- /dev/null +++ b/test/integ/ondemand-microsoftdynamics365-to-s3.integ.snapshot/manifest.json @@ -0,0 +1,95 @@ +{ + "version": "36.0.0", + "artifacts": { + "TestStack.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "TestStack.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "TestStack": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/us-east-1", + "properties": { + "templateFile": "TestStack.template.json", + "terminationProtection": false, + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-us-east-1", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-us-east-1", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-us-east-1/2c0e6f8d668b207ae36fefe56aefb498b434801e17454f6dbe84cdaf18c5bc6a.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "TestStack.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-us-east-1", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "TestStack.assets" + ], + "metadata": { + "/TestStack/TestConnector/TestConnector": [ + { + "type": "aws:cdk:logicalId", + "data": "TestConnectorEE9F0A13" + } + ], + "/TestStack/TestBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "TestBucket560B80BC" + } + ], + "/TestStack/TestBucket/Policy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "TestBucketPolicyBA12ED38" + } + ], + "/TestStack/TestBucket/AutoDeleteObjectsCustomResource/Default": [ + { + "type": "aws:cdk:logicalId", + "data": "TestBucketAutoDeleteObjectsCustomResource8FEAABD5" + } + ], + "/TestStack/Custom::S3AutoDeleteObjectsCustomResourceProvider/Role": [ + { + "type": "aws:cdk:logicalId", + "data": "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" + } + ], + "/TestStack/Custom::S3AutoDeleteObjectsCustomResourceProvider/Handler": [ + { + "type": "aws:cdk:logicalId", + "data": "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F" + } + ], + "/TestStack/OnDemandFlow/OnDemandFlow": [ + { + "type": "aws:cdk:logicalId", + "data": "OnDemandFlow4ECA54C5" + } + ], + "/TestStack/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/TestStack/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "TestStack" + } + } +} \ No newline at end of file diff --git a/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.assets.json index a2cca018..8e557ee6 100644 --- a/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.assets.json +++ b/test/integ/ondemand-microsoftsharepointonline-to-s3.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { diff --git a/test/integ/ondemand-s3-to-snowflake.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-s3-to-snowflake.integ.snapshot/TestStack.assets.json index c26b3f3b..0e8fc6e2 100644 --- a/test/integ/ondemand-s3-to-snowflake.integ.snapshot/TestStack.assets.json +++ b/test/integ/ondemand-s3-to-snowflake.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { diff --git a/test/integ/ondemand-salesforce-to-redshift.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-salesforce-to-redshift.integ.snapshot/TestStack.assets.json index 8ec931e7..4811a185 100644 --- a/test/integ/ondemand-salesforce-to-redshift.integ.snapshot/TestStack.assets.json +++ b/test/integ/ondemand-salesforce-to-redshift.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { @@ -14,15 +14,15 @@ } } }, - "cfac6ca3a4e2d5bd3b1972146b75664796ce0dc75ee0af8a149b3029ad745b3d": { + "6bdd909f81c84ffe7d00cf4d6a2dbac8606429bcc05b0db3da842c1941a532f2": { "source": { - "path": "asset.cfac6ca3a4e2d5bd3b1972146b75664796ce0dc75ee0af8a149b3029ad745b3d", + "path": "asset.6bdd909f81c84ffe7d00cf4d6a2dbac8606429bcc05b0db3da842c1941a532f2", "packaging": "zip" }, "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "cfac6ca3a4e2d5bd3b1972146b75664796ce0dc75ee0af8a149b3029ad745b3d.zip", + "objectKey": "6bdd909f81c84ffe7d00cf4d6a2dbac8606429bcc05b0db3da842c1941a532f2.zip", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } @@ -40,20 +40,20 @@ } } }, - "17c16a3854838fd3ff4bda08146122a6701f33b9c86ae17f415ad0dc47a97544": { + "ed6cd104ff5f101d06dae8cb2b87cc6e6d69b9a22055b467ea6cae10ff023023": { "source": { - "path": "asset.17c16a3854838fd3ff4bda08146122a6701f33b9c86ae17f415ad0dc47a97544", + "path": "asset.ed6cd104ff5f101d06dae8cb2b87cc6e6d69b9a22055b467ea6cae10ff023023", "packaging": "zip" }, "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "17c16a3854838fd3ff4bda08146122a6701f33b9c86ae17f415ad0dc47a97544.zip", + "objectKey": "ed6cd104ff5f101d06dae8cb2b87cc6e6d69b9a22055b467ea6cae10ff023023.zip", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } }, - "05538986ff9dd0b7f69751036b284217af71accf2c69f766facbfb9617a2afb9": { + "ad6641ba06a646432fa289b98f36bbe1f87c1aec7d517520d44ab10cb2fd7c30": { "source": { "path": "TestStack.template.json", "packaging": "file" @@ -61,7 +61,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "05538986ff9dd0b7f69751036b284217af71accf2c69f766facbfb9617a2afb9.json", + "objectKey": "ad6641ba06a646432fa289b98f36bbe1f87c1aec7d517520d44ab10cb2fd7c30.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/test/integ/ondemand-salesforce-to-redshift.integ.snapshot/TestStack.template.json b/test/integ/ondemand-salesforce-to-redshift.integ.snapshot/TestStack.template.json index 82267515..b75ce5c3 100644 --- a/test/integ/ondemand-salesforce-to-redshift.integ.snapshot/TestStack.template.json +++ b/test/integ/ondemand-salesforce-to-redshift.integ.snapshot/TestStack.template.json @@ -959,7 +959,7 @@ "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "S3Key": "cfac6ca3a4e2d5bd3b1972146b75664796ce0dc75ee0af8a149b3029ad745b3d.zip" + "S3Key": "6bdd909f81c84ffe7d00cf4d6a2dbac8606429bcc05b0db3da842c1941a532f2.zip" }, "Handler": "index.handler", "Role": { @@ -1237,7 +1237,7 @@ "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "S3Key": "17c16a3854838fd3ff4bda08146122a6701f33b9c86ae17f415ad0dc47a97544.zip" + "S3Key": "ed6cd104ff5f101d06dae8cb2b87cc6e6d69b9a22055b467ea6cae10ff023023.zip" }, "Handler": "index.handler", "Role": { diff --git a/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.assets.json index 3840dff8..205f1cf7 100644 --- a/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.assets.json +++ b/test/integ/ondemand-salesforce-to-s3.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { diff --git a/test/integ/ondemand-sapodata-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-sapodata-to-s3.integ.snapshot/TestStack.assets.json index 67fde2ef..9b73ea5e 100644 --- a/test/integ/ondemand-sapodata-to-s3.integ.snapshot/TestStack.assets.json +++ b/test/integ/ondemand-sapodata-to-s3.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { diff --git a/test/integ/ondemand-slack-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/ondemand-slack-to-s3.integ.snapshot/TestStack.assets.json index 1d031dd0..c6d3fb37 100644 --- a/test/integ/ondemand-slack-to-s3.integ.snapshot/TestStack.assets.json +++ b/test/integ/ondemand-slack-to-s3.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { diff --git a/test/integ/onevent-salesforce-to-eventbridge.integ.snapshot/TestStack.assets.json b/test/integ/onevent-salesforce-to-eventbridge.integ.snapshot/TestStack.assets.json index bb50ca4f..a662d522 100644 --- a/test/integ/onevent-salesforce-to-eventbridge.integ.snapshot/TestStack.assets.json +++ b/test/integ/onevent-salesforce-to-eventbridge.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { diff --git a/test/integ/onschedule-s3-to-salesforce.integ.snapshot/TestStack.assets.json b/test/integ/onschedule-s3-to-salesforce.integ.snapshot/TestStack.assets.json index 512bdad9..10af13c6 100644 --- a/test/integ/onschedule-s3-to-salesforce.integ.snapshot/TestStack.assets.json +++ b/test/integ/onschedule-s3-to-salesforce.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { diff --git a/test/integ/onschedule-salesforce-marketing-cloud-to-s3.integ.snapshot/TestStack.assets.json b/test/integ/onschedule-salesforce-marketing-cloud-to-s3.integ.snapshot/TestStack.assets.json index da14bf36..c6ff351e 100644 --- a/test/integ/onschedule-salesforce-marketing-cloud-to-s3.integ.snapshot/TestStack.assets.json +++ b/test/integ/onschedule-salesforce-marketing-cloud-to-s3.integ.snapshot/TestStack.assets.json @@ -1,5 +1,5 @@ { - "version": "35.0.0", + "version": "36.0.0", "files": { "b7f33614a69548d6bafe224d751a7ef238cde19097415e553fe8b63a4c8fd8a6": { "source": { diff --git a/tsconfig.dev.json b/tsconfig.dev.json index 88cc2f2f..da832d0d 100644 --- a/tsconfig.dev.json +++ b/tsconfig.dev.json @@ -26,7 +26,6 @@ "target": "ES2019" }, "include": [ - ".projenrc.js", "src/**/*.ts", "test/**/*.ts", ".projenrc.ts", diff --git a/yarn.lock b/yarn.lock index 5a1bfd4c..8166f198 100644 --- a/yarn.lock +++ b/yarn.lock @@ -30,15 +30,15 @@ resolved "https://registry.yarnpkg.com/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.0.1.tgz#6dc9b7cdb22ff622a7176141197962360c33e9ac" integrity sha512-DDt4SLdLOwWCjGtltH4VCST7hpOI5DzieuhGZsBpZ+AgJdSI2GCjklCXm0GCTwJG/SolkL5dtQXyUKgg9luBDg== -"@aws-cdk/aws-glue-alpha@2.116.1-alpha.0": - version "2.116.1-alpha.0" - resolved "https://registry.yarnpkg.com/@aws-cdk/aws-glue-alpha/-/aws-glue-alpha-2.116.1-alpha.0.tgz#9ea440b0e90b172d5c97a9afddef0206f4174142" - integrity sha512-U0BawxewRqmBgm03NTEtDkPiSO5GpuLG0xW+g78DudDnzdW6qcJmO/oOz6qoUEbR24VK8oqYqOXXoYzaHVWvyw== +"@aws-cdk/aws-glue-alpha@2.121.1-alpha.0": + version "2.121.1-alpha.0" + resolved "https://registry.yarnpkg.com/@aws-cdk/aws-glue-alpha/-/aws-glue-alpha-2.121.1-alpha.0.tgz#8c5e6b34ccfd35b7512e84c2e6176e88a1482a80" + integrity sha512-ovZeXXCNBCkRdHEfBCbDYOZ1tTbwlV1Njc1iPwP7aMo4BIMSrRzLF5ir+InwKbnRA1FWACyDGXRakue/fuT4aA== -"@aws-cdk/aws-redshift-alpha@2.116.1-alpha.0": - version "2.116.1-alpha.0" - resolved "https://registry.yarnpkg.com/@aws-cdk/aws-redshift-alpha/-/aws-redshift-alpha-2.116.1-alpha.0.tgz#1b67d00893e43c8e1cc212db67e2c0f5bfe0bf64" - integrity sha512-/jG35//7U5CQx3pTzeWbls3bbZ0ilPaBqPgfFSBz/WMi9ToLkYth4eHBeFfhMzeBs5sLKUSB4OBlqFJYXsoWeg== +"@aws-cdk/aws-redshift-alpha@2.121.1-alpha.0": + version "2.121.1-alpha.0" + resolved "https://registry.yarnpkg.com/@aws-cdk/aws-redshift-alpha/-/aws-redshift-alpha-2.121.1-alpha.0.tgz#21eae9ef3843290919f1e637822b3ab8a74bf1d7" + integrity sha512-PsCtr+IuWoyJgi40iVLCB65kiWOpzI6iG/jlOl9lQkQvDO5Dh19OOtRSTVIoFga6AbcVAVDmhQ/GyEHBWoELZg== "@aws-cdk/aws-service-spec@0.0.29": version "0.0.29" @@ -1354,10 +1354,10 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== -aws-cdk-lib@2.116.1: - version "2.116.1" - resolved "https://registry.yarnpkg.com/aws-cdk-lib/-/aws-cdk-lib-2.116.1.tgz#4810ae231b1b87332a158b0b15a7d85dd5bc9b02" - integrity sha512-3j9kUHZFp/Jl79utnsBB2xW2u+sNfjpliQtTPQlW0i7J12oxjD2rHU4AJV741cKMxnA9VrHnRyJF9NTryji8yw== +aws-cdk-lib@2.121.1: + version "2.121.1" + resolved "https://registry.yarnpkg.com/aws-cdk-lib/-/aws-cdk-lib-2.121.1.tgz#c4c5029a916fdbc9ae8ed4fd39264143d6326322" + integrity sha512-wrOHDDQqVuH2tRTH7p2LaMPVI3V2bqr8X//Qrhy+hOFA04u2MVetQYev+NgMXPjNzU9IGX2wy1Fs9Y/ntq2sQA== dependencies: "@aws-cdk/asset-awscli-v1" "^2.2.201" "@aws-cdk/asset-kubectl-v20" "^2.1.2" @@ -1552,10 +1552,10 @@ case@1.6.3, case@^1.6.3: resolved "https://registry.yarnpkg.com/case/-/case-1.6.3.tgz#0a4386e3e9825351ca2e6216c60467ff5f1ea1c9" integrity sha512-mzDSXIPaFwVDvZAHqZ9VlbyF4yyXRuX6IvB06WvPYkqJVO24kX1PPhv9bfpKNFZyxYFmmgo03HUiD8iklmJYRQ== -cdklabs-projen-project-types@^0.1.184: - version "0.1.184" - resolved "https://registry.yarnpkg.com/cdklabs-projen-project-types/-/cdklabs-projen-project-types-0.1.184.tgz#b5d6fb7ddcb2ffd71d1ecccd723ffe54f0db14a5" - integrity sha512-saF/295s0Pzrk9/N9S7MHDtWWJaK8nYxl68xLQe15sWPitVjCp6hUgEfZh8TWeIpTnXh3N4btZLxSGjUspVS4A== +cdklabs-projen-project-types@^0.1.186: + version "0.1.186" + resolved "https://registry.yarnpkg.com/cdklabs-projen-project-types/-/cdklabs-projen-project-types-0.1.186.tgz#7bf27240000b86d7b495df62abbc393606e96677" + integrity sha512-EF1DMNFFsP+BH/eTB8/Qm8qSd6Sok50CKNrit1tIltJGYOqfJy7CciSrY30NZPDar+/cSgdDv5F2iUs6neW0dw== dependencies: yaml "^2.3.4" @@ -4514,10 +4514,10 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== -projen@^0.77.6: - version "0.77.6" - resolved "https://registry.yarnpkg.com/projen/-/projen-0.77.6.tgz#480c1af4246bba55e8d0732ff7a9d1804ce414be" - integrity sha512-nXbbDr81UjfLjCfVfHGfGPIjiN7INSyMUa52FYupX0TmybMq+CnvX8o0O45feOLLhsifNq7EHXtF+hgBtpBb8A== +projen@^0.79.1: + version "0.79.1" + resolved "https://registry.yarnpkg.com/projen/-/projen-0.79.1.tgz#bff39a8378675dc92d7a880f8f57595b52f51bcd" + integrity sha512-BkkCdGXZ+1pCN6hUZadxclweDfFkH6k1uAUpoY+B4awDDCtQQ7A9/Is6zyc89Jt38xDg7Ctc+/1Z86WwjOIENA== dependencies: "@iarna/toml" "^2.2.5" case "^1.6.3"