From 6e7e00eb87d39fae51d660ff78e2400f8bae7556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Paw=C5=82aszek?= Date: Wed, 17 Jan 2024 14:14:34 +0100 Subject: [PATCH] chore: merged with main and upgraded deps --- .github/workflows/release.yml | 19 +- .gitignore | 2 +- .npmignore | 2 + .projen/tasks.json | 3 +- API.md | 900 +++++++++++++++++++++++++--- package.json | 4 +- src/googlebigquery/profile.ts | 17 +- src/index.ts | 1 + test/googlebigquery/profile.test.ts | 286 +++++++++ test/googlebigquery/source.test.ts | 224 +++++++ tsconfig.dev.json | 1 - yarn.lock | 16 +- 12 files changed, 1352 insertions(+), 123 deletions(-) create mode 100644 test/googlebigquery/profile.test.ts create mode 100644 test/googlebigquery/source.test.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 916aa39e..9a2e5fb5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,7 @@ jobs: contents: write outputs: latest_commit: ${{ steps.git_remote.outputs.latest_commit }} + tag_exists: ${{ steps.check_tag_exists.outputs.exists }} env: CI: "true" steps: @@ -32,6 +33,12 @@ jobs: run: yarn install --check-files --frozen-lockfile - name: release run: npx projen release + - name: Check if releasetag already exists + id: check_tag_exists + run: |- + TAG=$(cat dist/dist/releasetag.txt) + ([ ! -z "$TAG" ] && git ls-remote -q --exit-code --tags origin $TAG && (echo "exists=true" >> $GITHUB_OUTPUT)) || echo "exists=false" >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT - name: Check for new commits id: git_remote run: echo "latest_commit=$(git ls-remote origin -h ${{ github.ref }} | cut -f1)" >> $GITHUB_OUTPUT @@ -51,7 +58,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-node@v3 with: @@ -80,7 +87,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-node@v3 with: @@ -113,7 +120,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-java@v3 with: @@ -153,7 +160,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-node@v3 with: @@ -188,7 +195,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-node@v3 with: @@ -222,7 +229,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - if: needs.release.outputs.latest_commit == github.sha + if: needs.release.outputs.tag_exists != 'true' && needs.release.outputs.latest_commit == github.sha steps: - uses: actions/setup-node@v3 with: diff --git a/.gitignore b/.gitignore index 7525aee2..1df3781d 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ jspm_packages/ *.rest .vscode **/.DS_Store -!/.projenrc.js /test-reports/ junit.xml /coverage/ @@ -165,3 +164,4 @@ test/integ/onschedule-salesforce-marketing-cloud-to-s3.integ.snapshot/**/tree.js !/.github/workflows/upgrade-cdklabs-projen-project-types-main.yml !/.github/workflows/upgrade-main.yml !/.github/workflows/upgrade-dev-deps-main.yml +!/.projenrc.ts diff --git a/.npmignore b/.npmignore index 7b7315e8..fdaa448c 100644 --- a/.npmignore +++ b/.npmignore @@ -37,3 +37,5 @@ test/integ/onevent-salesforce-to-eventbridge.integ.snapshot test/integ/onschedule-s3-to-salesforce.integ.snapshot test/integ/.tmp test/integ/onschedule-salesforce-marketing-cloud-to-s3.integ.snapshot +/.gitattributes +/.projenrc.ts diff --git a/.projen/tasks.json b/.projen/tasks.json index 7ba967ab..d5e52525 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -154,7 +154,8 @@ "description": "Runs eslint against the codebase", "steps": [ { - "exec": "eslint --ext .ts,.tsx --fix --no-error-on-unmatched-pattern src test build-tools projenrc .projenrc.ts" + "exec": "eslint --ext .ts,.tsx --fix --no-error-on-unmatched-pattern $@ src test build-tools projenrc .projenrc.ts", + "receiveArgs": true } ] }, diff --git a/API.md b/API.md index 1f3fd1c4..7e69e6e6 100644 --- a/API.md +++ b/API.md @@ -1447,6 +1447,282 @@ public readonly credentials: ISecret; --- +### GoogleBigQueryConnectorProfile + +#### Initializers + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +new GoogleBigQueryConnectorProfile(scope: Construct, id: string, props: GoogleBigQueryConnectorProfileProps) +``` + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| scope | constructs.Construct | *No description.* | +| id | string | *No description.* | +| props | GoogleBigQueryConnectorProfileProps | *No description.* | + +--- + +##### `scope`Required + +- *Type:* constructs.Construct + +--- + +##### `id`Required + +- *Type:* string + +--- + +##### `props`Required + +- *Type:* GoogleBigQueryConnectorProfileProps + +--- + +#### Methods + +| **Name** | **Description** | +| --- | --- | +| toString | Returns a string representation of this construct. | +| applyRemovalPolicy | Apply the given removal policy to this resource. | + +--- + +##### `toString` + +```typescript +public toString(): string +``` + +Returns a string representation of this construct. + +##### `applyRemovalPolicy` + +```typescript +public applyRemovalPolicy(policy: RemovalPolicy): void +``` + +Apply the given removal policy to this resource. + +The Removal Policy controls what happens to this resource when it stops +being managed by CloudFormation, either because you've removed it from the +CDK application or because you've made a change that requires the resource +to be replaced. + +The resource can be deleted (`RemovalPolicy.DESTROY`), or left in your AWS +account for data recovery and cleanup later (`RemovalPolicy.RETAIN`). + +###### `policy`Required + +- *Type:* aws-cdk-lib.RemovalPolicy + +--- + +#### Static Functions + +| **Name** | **Description** | +| --- | --- | +| isConstruct | Checks if `x` is a construct. | +| isOwnedResource | Returns true if the construct was created by CDK, and false otherwise. | +| isResource | Check whether the given construct is a Resource. | +| fromConnectionProfileArn | *No description.* | +| fromConnectionProfileName | *No description.* | + +--- + +##### ~~`isConstruct`~~ + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +GoogleBigQueryConnectorProfile.isConstruct(x: any) +``` + +Checks if `x` is a construct. + +###### `x`Required + +- *Type:* any + +Any object. + +--- + +##### `isOwnedResource` + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +GoogleBigQueryConnectorProfile.isOwnedResource(construct: IConstruct) +``` + +Returns true if the construct was created by CDK, and false otherwise. + +###### `construct`Required + +- *Type:* constructs.IConstruct + +--- + +##### `isResource` + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +GoogleBigQueryConnectorProfile.isResource(construct: IConstruct) +``` + +Check whether the given construct is a Resource. + +###### `construct`Required + +- *Type:* constructs.IConstruct + +--- + +##### `fromConnectionProfileArn` + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +GoogleBigQueryConnectorProfile.fromConnectionProfileArn(scope: Construct, id: string, arn: string) +``` + +###### `scope`Required + +- *Type:* constructs.Construct + +--- + +###### `id`Required + +- *Type:* string + +--- + +###### `arn`Required + +- *Type:* string + +--- + +##### `fromConnectionProfileName` + +```typescript +import { GoogleBigQueryConnectorProfile } from '@cdklabs/cdk-appflow' + +GoogleBigQueryConnectorProfile.fromConnectionProfileName(scope: Construct, id: string, name: string) +``` + +###### `scope`Required + +- *Type:* constructs.Construct + +--- + +###### `id`Required + +- *Type:* string + +--- + +###### `name`Required + +- *Type:* string + +--- + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| node | constructs.Node | The tree node. | +| env | aws-cdk-lib.ResourceEnvironment | The environment this resource belongs to. | +| stack | aws-cdk-lib.Stack | The stack in which this resource is defined. | +| arn | string | *No description.* | +| name | string | *No description.* | +| credentials | aws-cdk-lib.aws_secretsmanager.ISecret | *No description.* | + +--- + +##### `node`Required + +```typescript +public readonly node: Node; +``` + +- *Type:* constructs.Node + +The tree node. + +--- + +##### `env`Required + +```typescript +public readonly env: ResourceEnvironment; +``` + +- *Type:* aws-cdk-lib.ResourceEnvironment + +The environment this resource belongs to. + +For resources that are created and managed by the CDK +(generally, those created by creating new class instances like Role, Bucket, etc.), +this is always the same as the environment of the stack they belong to; +however, for imported resources +(those obtained from static methods like fromRoleArn, fromBucketName, etc.), +that might be different than the stack they were imported into. + +--- + +##### `stack`Required + +```typescript +public readonly stack: Stack; +``` + +- *Type:* aws-cdk-lib.Stack + +The stack in which this resource is defined. + +--- + +##### `arn`Required + +```typescript +public readonly arn: string; +``` + +- *Type:* string + +--- + +##### `name`Required + +```typescript +public readonly name: string; +``` + +- *Type:* string + +--- + +##### `credentials`Optional + +```typescript +public readonly credentials: ISecret; +``` + +- *Type:* aws-cdk-lib.aws_secretsmanager.ISecret + +--- + + ### JdbcSmallDataScaleConnectorProfile The connector profile for the JDBC connector. @@ -7056,111 +7332,410 @@ const flowProps: FlowProps = { ... } public readonly destination: IDestination; ``` -- *Type:* IDestination +- *Type:* IDestination + +--- + +##### `mappings`Required + +```typescript +public readonly mappings: IMapping[]; +``` + +- *Type:* IMapping[] + +--- + +##### `source`Required + +```typescript +public readonly source: ISource; +``` + +- *Type:* ISource + +--- + +##### `description`Optional + +```typescript +public readonly description: string; +``` + +- *Type:* string + +--- + +##### `filters`Optional + +```typescript +public readonly filters: IFilter[]; +``` + +- *Type:* IFilter[] + +--- + +##### `key`Optional + +```typescript +public readonly key: IKey; +``` + +- *Type:* aws-cdk-lib.aws_kms.IKey + +--- + +##### `name`Optional + +```typescript +public readonly name: string; +``` + +- *Type:* string + +--- + +##### `transforms`Optional + +```typescript +public readonly transforms: ITransform[]; +``` + +- *Type:* ITransform[] + +--- + +##### `validations`Optional + +```typescript +public readonly validations: IValidation[]; +``` + +- *Type:* IValidation[] + +--- + +### GoogleAnalytics4ConnectorProfileProps + +#### Initializer + +```typescript +import { GoogleAnalytics4ConnectorProfileProps } from '@cdklabs/cdk-appflow' + +const googleAnalytics4ConnectorProfileProps: GoogleAnalytics4ConnectorProfileProps = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| key | aws-cdk-lib.aws_kms.IKey | TODO: think if this should be here as not all connector profiles have that. | +| name | string | *No description.* | +| oAuth | GoogleAnalytics4OAuthSettings | *No description.* | + +--- + +##### `key`Optional + +```typescript +public readonly key: IKey; +``` + +- *Type:* aws-cdk-lib.aws_kms.IKey + +TODO: think if this should be here as not all connector profiles have that. + +--- + +##### `name`Optional + +```typescript +public readonly name: string; +``` + +- *Type:* string + +--- + +##### `oAuth`Required + +```typescript +public readonly oAuth: GoogleAnalytics4OAuthSettings; +``` + +- *Type:* GoogleAnalytics4OAuthSettings + +--- + +### GoogleAnalytics4OAuthEndpoints + +Google's OAuth token and authorization endpoints. + +#### Initializer + +```typescript +import { GoogleAnalytics4OAuthEndpoints } from '@cdklabs/cdk-appflow' + +const googleAnalytics4OAuthEndpoints: GoogleAnalytics4OAuthEndpoints = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| authorization | string | The OAuth authorization endpoint URI. | +| token | string | The OAuth token endpoint URI. | + +--- + +##### `authorization`Optional + +```typescript +public readonly authorization: string; +``` + +- *Type:* string + +The OAuth authorization endpoint URI. + +--- + +##### `token`Optional + +```typescript +public readonly token: string; +``` + +- *Type:* string + +The OAuth token endpoint URI. + +--- + +### GoogleAnalytics4OAuthFlow + +Represents the OAuth flow enabled for the GA4. + +#### Initializer + +```typescript +import { GoogleAnalytics4OAuthFlow } from '@cdklabs/cdk-appflow' + +const googleAnalytics4OAuthFlow: GoogleAnalytics4OAuthFlow = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| refreshTokenGrant | GoogleAnalytics4RefreshTokenGrantFlow | The details required for executing the refresh token grant flow. | + +--- + +##### `refreshTokenGrant`Required + +```typescript +public readonly refreshTokenGrant: GoogleAnalytics4RefreshTokenGrantFlow; +``` + +- *Type:* GoogleAnalytics4RefreshTokenGrantFlow + +The details required for executing the refresh token grant flow. + +--- + +### GoogleAnalytics4OAuthSettings + +#### Initializer + +```typescript +import { GoogleAnalytics4OAuthSettings } from '@cdklabs/cdk-appflow' + +const googleAnalytics4OAuthSettings: GoogleAnalytics4OAuthSettings = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| accessToken | aws-cdk-lib.SecretValue | The access token to be used when interacting with Google Analytics 4. | +| endpoints | GoogleAnalytics4OAuthEndpoints | The OAuth token and authorization endpoints. | +| flow | GoogleAnalytics4OAuthFlow | The OAuth flow used for obtaining a new accessToken when the old is not present or expired. | + +--- + +##### `accessToken`Optional + +```typescript +public readonly accessToken: SecretValue; +``` + +- *Type:* aws-cdk-lib.SecretValue +- *Default:* Retrieves a fresh accessToken with the information in the [flow property]{@link GoogleAnalytics4OAuthSettings#flow } + +The access token to be used when interacting with Google Analytics 4. + +Note that if only the access token is provided AppFlow is not able to retrieve a fresh access token when the current one is expired + +--- + +##### `endpoints`Optional + +```typescript +public readonly endpoints: GoogleAnalytics4OAuthEndpoints; +``` + +- *Type:* GoogleAnalytics4OAuthEndpoints + +The OAuth token and authorization endpoints. + +--- + +##### `flow`Optional + +```typescript +public readonly flow: GoogleAnalytics4OAuthFlow; +``` + +- *Type:* GoogleAnalytics4OAuthFlow +- *Default:* undefined. AppFlow will not request any new accessToken after expiry. + +The OAuth flow used for obtaining a new accessToken when the old is not present or expired. --- -##### `mappings`Required +### GoogleAnalytics4RefreshTokenGrantFlow + +The OAuth elements required for the execution of the refresh token grant flow. + +#### Initializer ```typescript -public readonly mappings: IMapping[]; +import { GoogleAnalytics4RefreshTokenGrantFlow } from '@cdklabs/cdk-appflow' + +const googleAnalytics4RefreshTokenGrantFlow: GoogleAnalytics4RefreshTokenGrantFlow = { ... } ``` -- *Type:* IMapping[] +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| clientId | aws-cdk-lib.SecretValue | The id of the client app. | +| clientSecret | aws-cdk-lib.SecretValue | The secret of the client app. | +| refreshToken | aws-cdk-lib.SecretValue | A non-expired refresh token. | --- -##### `source`Required +##### `clientId`Optional ```typescript -public readonly source: ISource; +public readonly clientId: SecretValue; ``` -- *Type:* ISource +- *Type:* aws-cdk-lib.SecretValue + +The id of the client app. --- -##### `description`Optional +##### `clientSecret`Optional ```typescript -public readonly description: string; +public readonly clientSecret: SecretValue; ``` -- *Type:* string +- *Type:* aws-cdk-lib.SecretValue + +The secret of the client app. --- -##### `filters`Optional +##### `refreshToken`Optional ```typescript -public readonly filters: IFilter[]; +public readonly refreshToken: SecretValue; ``` -- *Type:* IFilter[] +- *Type:* aws-cdk-lib.SecretValue + +A non-expired refresh token. --- -##### `key`Optional +### GoogleAnalytics4SourceProps + +Properties of a Google Analytics v4 Source. + +#### Initializer ```typescript -public readonly key: IKey; +import { GoogleAnalytics4SourceProps } from '@cdklabs/cdk-appflow' + +const googleAnalytics4SourceProps: GoogleAnalytics4SourceProps = { ... } ``` -- *Type:* aws-cdk-lib.aws_kms.IKey +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| apiVersion | string | *No description.* | +| object | string | *No description.* | +| profile | GoogleAnalytics4ConnectorProfile | *No description.* | --- -##### `name`Optional +##### `apiVersion`Required ```typescript -public readonly name: string; +public readonly apiVersion: string; ``` - *Type:* string --- -##### `transforms`Optional +##### `object`Required ```typescript -public readonly transforms: ITransform[]; +public readonly object: string; ``` -- *Type:* ITransform[] +- *Type:* string --- -##### `validations`Optional +##### `profile`Required ```typescript -public readonly validations: IValidation[]; +public readonly profile: GoogleAnalytics4ConnectorProfile; ``` -- *Type:* IValidation[] +- *Type:* GoogleAnalytics4ConnectorProfile --- -### GoogleAnalytics4ConnectorProfileProps +### GoogleBigQueryConnectorProfileProps -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4ConnectorProfileProps } from '@cdklabs/cdk-appflow' +import { GoogleBigQueryConnectorProfileProps } from '@cdklabs/cdk-appflow' -const googleAnalytics4ConnectorProfileProps: GoogleAnalytics4ConnectorProfileProps = { ... } +const googleBigQueryConnectorProfileProps: GoogleBigQueryConnectorProfileProps = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| key | aws-cdk-lib.aws_kms.IKey | TODO: think if this should be here as not all connector profiles have that. | -| name | string | *No description.* | -| oAuth | GoogleAnalytics4OAuthSettings | *No description.* | +| key | aws-cdk-lib.aws_kms.IKey | TODO: think if this should be here as not all connector profiles have that. | +| name | string | *No description.* | +| oAuth | GoogleBigQueryOAuthSettings | *No description.* | --- -##### `key`Optional +##### `key`Optional ```typescript public readonly key: IKey; @@ -7172,7 +7747,7 @@ TODO: think if this should be here as not all connector profiles have that. --- -##### `name`Optional +##### `name`Optional ```typescript public readonly name: string; @@ -7182,38 +7757,38 @@ public readonly name: string; --- -##### `oAuth`Required +##### `oAuth`Required ```typescript -public readonly oAuth: GoogleAnalytics4OAuthSettings; +public readonly oAuth: GoogleBigQueryOAuthSettings; ``` -- *Type:* GoogleAnalytics4OAuthSettings +- *Type:* GoogleBigQueryOAuthSettings --- -### GoogleAnalytics4OAuthEndpoints +### GoogleBigQueryOAuthEndpoints Google's OAuth token and authorization endpoints. -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4OAuthEndpoints } from '@cdklabs/cdk-appflow' +import { GoogleBigQueryOAuthEndpoints } from '@cdklabs/cdk-appflow' -const googleAnalytics4OAuthEndpoints: GoogleAnalytics4OAuthEndpoints = { ... } +const googleBigQueryOAuthEndpoints: GoogleBigQueryOAuthEndpoints = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| authorization | string | The OAuth authorization endpoint URI. | -| token | string | The OAuth token endpoint URI. | +| authorization | string | The OAuth authorization endpoint URI. | +| token | string | The OAuth token endpoint URI. | --- -##### `authorization`Optional +##### `authorization`Optional ```typescript public readonly authorization: string; @@ -7225,7 +7800,7 @@ The OAuth authorization endpoint URI. --- -##### `token`Optional +##### `token`Optional ```typescript public readonly token: string; @@ -7237,121 +7812,171 @@ The OAuth token endpoint URI. --- -### GoogleAnalytics4OAuthFlow +### GoogleBigQueryOAuthFlow Represents the OAuth flow enabled for the GA4. -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4OAuthFlow } from '@cdklabs/cdk-appflow' +import { GoogleBigQueryOAuthFlow } from '@cdklabs/cdk-appflow' -const googleAnalytics4OAuthFlow: GoogleAnalytics4OAuthFlow = { ... } +const googleBigQueryOAuthFlow: GoogleBigQueryOAuthFlow = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| refreshTokenGrant | GoogleAnalytics4RefreshTokenGrantFlow | The details required for executing the refresh token grant flow. | +| refreshTokenGrant | GoogleBigQueryRefreshTokenGrantFlow | The details required for executing the refresh token grant flow. | --- -##### `refreshTokenGrant`Required +##### `refreshTokenGrant`Required ```typescript -public readonly refreshTokenGrant: GoogleAnalytics4RefreshTokenGrantFlow; +public readonly refreshTokenGrant: GoogleBigQueryRefreshTokenGrantFlow; ``` -- *Type:* GoogleAnalytics4RefreshTokenGrantFlow +- *Type:* GoogleBigQueryRefreshTokenGrantFlow The details required for executing the refresh token grant flow. --- -### GoogleAnalytics4OAuthSettings +### GoogleBigQueryOAuthSettings -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4OAuthSettings } from '@cdklabs/cdk-appflow' +import { GoogleBigQueryOAuthSettings } from '@cdklabs/cdk-appflow' -const googleAnalytics4OAuthSettings: GoogleAnalytics4OAuthSettings = { ... } +const googleBigQueryOAuthSettings: GoogleBigQueryOAuthSettings = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| accessToken | aws-cdk-lib.SecretValue | The access token to be used when interacting with Google Analytics 4. | -| endpoints | GoogleAnalytics4OAuthEndpoints | The OAuth token and authorization endpoints. | -| flow | GoogleAnalytics4OAuthFlow | The OAuth flow used for obtaining a new accessToken when the old is not present or expired. | +| accessToken | aws-cdk-lib.SecretValue | The access token to be used when interacting with Google BigQuery. | +| endpoints | GoogleBigQueryOAuthEndpoints | The OAuth token and authorization endpoints. | +| flow | GoogleBigQueryOAuthFlow | The OAuth flow used for obtaining a new accessToken when the old is not present or expired. | --- -##### `accessToken`Optional +##### `accessToken`Optional ```typescript public readonly accessToken: SecretValue; ``` - *Type:* aws-cdk-lib.SecretValue -- *Default:* Retrieves a fresh accessToken with the information in the [flow property]{@link GoogleAnalytics4OAuthSettings#flow } +- *Default:* Retrieves a fresh accessToken with the information in the [flow property]{@link GoogleBigQueryOAuthSettings#flow } -The access token to be used when interacting with Google Analytics 4. +The access token to be used when interacting with Google BigQuery. Note that if only the access token is provided AppFlow is not able to retrieve a fresh access token when the current one is expired --- -##### `endpoints`Optional +##### `endpoints`Optional ```typescript -public readonly endpoints: GoogleAnalytics4OAuthEndpoints; +public readonly endpoints: GoogleBigQueryOAuthEndpoints; ``` -- *Type:* GoogleAnalytics4OAuthEndpoints +- *Type:* GoogleBigQueryOAuthEndpoints The OAuth token and authorization endpoints. --- -##### `flow`Optional +##### `flow`Optional ```typescript -public readonly flow: GoogleAnalytics4OAuthFlow; +public readonly flow: GoogleBigQueryOAuthFlow; ``` -- *Type:* GoogleAnalytics4OAuthFlow +- *Type:* GoogleBigQueryOAuthFlow - *Default:* undefined. AppFlow will not request any new accessToken after expiry. The OAuth flow used for obtaining a new accessToken when the old is not present or expired. --- -### GoogleAnalytics4RefreshTokenGrantFlow +### GoogleBigQueryObject + +#### Initializer + +```typescript +import { GoogleBigQueryObject } from '@cdklabs/cdk-appflow' + +const googleBigQueryObject: GoogleBigQueryObject = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| dataset | string | *No description.* | +| project | string | *No description.* | +| table | string | *No description.* | + +--- + +##### `dataset`Required + +```typescript +public readonly dataset: string; +``` + +- *Type:* string + +--- + +##### `project`Required + +```typescript +public readonly project: string; +``` + +- *Type:* string + +--- + +##### `table`Required + +```typescript +public readonly table: string; +``` + +- *Type:* string + +--- + +### GoogleBigQueryRefreshTokenGrantFlow The OAuth elements required for the execution of the refresh token grant flow. -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4RefreshTokenGrantFlow } from '@cdklabs/cdk-appflow' +import { GoogleBigQueryRefreshTokenGrantFlow } from '@cdklabs/cdk-appflow' -const googleAnalytics4RefreshTokenGrantFlow: GoogleAnalytics4RefreshTokenGrantFlow = { ... } +const googleBigQueryRefreshTokenGrantFlow: GoogleBigQueryRefreshTokenGrantFlow = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| clientId | aws-cdk-lib.SecretValue | The id of the client app. | -| clientSecret | aws-cdk-lib.SecretValue | The secret of the client app. | -| refreshToken | aws-cdk-lib.SecretValue | A non-expired refresh token. | +| clientId | aws-cdk-lib.SecretValue | The id of the client app. | +| clientSecret | aws-cdk-lib.SecretValue | The secret of the client app. | +| refreshToken | aws-cdk-lib.SecretValue | A non-expired refresh token. | --- -##### `clientId`Optional +##### `clientId`Optional ```typescript public readonly clientId: SecretValue; @@ -7363,7 +7988,7 @@ The id of the client app. --- -##### `clientSecret`Optional +##### `clientSecret`Optional ```typescript public readonly clientSecret: SecretValue; @@ -7375,7 +8000,7 @@ The secret of the client app. --- -##### `refreshToken`Optional +##### `refreshToken`Optional ```typescript public readonly refreshToken: SecretValue; @@ -7387,29 +8012,29 @@ A non-expired refresh token. --- -### GoogleAnalytics4SourceProps +### GoogleBigQuerySourceProps -Properties of a Google Analytics v4 Source. +Properties of a Google BigQuery Source. -#### Initializer +#### Initializer ```typescript -import { GoogleAnalytics4SourceProps } from '@cdklabs/cdk-appflow' +import { GoogleBigQuerySourceProps } from '@cdklabs/cdk-appflow' -const googleAnalytics4SourceProps: GoogleAnalytics4SourceProps = { ... } +const googleBigQuerySourceProps: GoogleBigQuerySourceProps = { ... } ``` #### Properties | **Name** | **Type** | **Description** | | --- | --- | --- | -| apiVersion | string | *No description.* | -| object | string | *No description.* | -| profile | GoogleAnalytics4ConnectorProfile | *No description.* | +| apiVersion | string | *No description.* | +| object | GoogleBigQueryObject | *No description.* | +| profile | GoogleBigQueryConnectorProfile | *No description.* | --- -##### `apiVersion`Required +##### `apiVersion`Required ```typescript public readonly apiVersion: string; @@ -7419,23 +8044,23 @@ public readonly apiVersion: string; --- -##### `object`Required +##### `object`Required ```typescript -public readonly object: string; +public readonly object: GoogleBigQueryObject; ``` -- *Type:* string +- *Type:* GoogleBigQueryObject --- -##### `profile`Required +##### `profile`Required ```typescript -public readonly profile: GoogleAnalytics4ConnectorProfile; +public readonly profile: GoogleBigQueryConnectorProfile; ``` -- *Type:* GoogleAnalytics4ConnectorProfile +- *Type:* GoogleBigQueryConnectorProfile --- @@ -12777,6 +13402,74 @@ The AppFlow type of the connector that this source is implemented for. --- +### GoogleBigQuerySource + +- *Implements:* ISource + +A class that represents a Google BigQuery Source. + +#### Initializers + +```typescript +import { GoogleBigQuerySource } from '@cdklabs/cdk-appflow' + +new GoogleBigQuerySource(props: GoogleBigQuerySourceProps) +``` + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| props | GoogleBigQuerySourceProps | *No description.* | + +--- + +##### `props`Required + +- *Type:* GoogleBigQuerySourceProps + +--- + +#### Methods + +| **Name** | **Description** | +| --- | --- | +| bind | *No description.* | + +--- + +##### `bind` + +```typescript +public bind(scope: IFlow): SourceFlowConfigProperty +``` + +###### `scope`Required + +- *Type:* IFlow + +--- + + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| connectorType | ConnectorType | The AppFlow type of the connector that this source is implemented for. | + +--- + +##### `connectorType`Required + +```typescript +public readonly connectorType: ConnectorType; +``` + +- *Type:* ConnectorType + +The AppFlow type of the connector that this source is implemented for. + +--- + + ### JdbcSmallDataScaleSource - *Implements:* ISource @@ -15184,7 +15877,7 @@ The AppFlow type of the connector that this source is implemented for. - *Extends:* aws-cdk-lib.IResource -- *Implemented By:* AmazonRdsForPostgreSqlConnectorProfile, ConnectorProfileBase, GoogleAnalytics4ConnectorProfile, JdbcSmallDataScaleConnectorProfile, MarketoConnectorProfile, MicrosoftDynamics365ConnectorProfile, MicrosoftSharepointOnlineConnectorProfile, RedshiftConnectorProfile, SAPOdataConnectorProfile, SalesforceConnectorProfile, SalesforceMarketingCloudConnectorProfile, ServiceNowConnectorProfile, SlackConnectorProfile, SnowflakeConnectorProfile, ZendeskConnectorProfile, IConnectorProfile +- *Implemented By:* AmazonRdsForPostgreSqlConnectorProfile, ConnectorProfileBase, GoogleAnalytics4ConnectorProfile, GoogleBigQueryConnectorProfile, JdbcSmallDataScaleConnectorProfile, MarketoConnectorProfile, MicrosoftDynamics365ConnectorProfile, MicrosoftSharepointOnlineConnectorProfile, RedshiftConnectorProfile, SAPOdataConnectorProfile, SalesforceConnectorProfile, SalesforceMarketingCloudConnectorProfile, ServiceNowConnectorProfile, SlackConnectorProfile, SnowflakeConnectorProfile, ZendeskConnectorProfile, IConnectorProfile #### Properties @@ -15596,7 +16289,7 @@ public bind(flow: IFlow, source: ISource): TaskProperty[] - *Extends:* IVertex -- *Implemented By:* GoogleAnalytics4Source, JdbcSmallDataScaleSource, MarketoSource, MicrosoftDynamics365Source, MicrosoftSharepointOnlineSource, S3Source, SAPOdataSource, SalesforceMarketingCloudSource, SalesforceSource, ServiceNowSource, SlackSource, ZendeskSource, ISource +- *Implemented By:* GoogleAnalytics4Source, GoogleBigQuerySource, JdbcSmallDataScaleSource, MarketoSource, MicrosoftDynamics365Source, MicrosoftSharepointOnlineSource, S3Source, SAPOdataSource, SalesforceMarketingCloudSource, SalesforceSource, ServiceNowSource, SlackSource, ZendeskSource, ISource A source of an AppFlow flow. @@ -15695,7 +16388,7 @@ A representation of a validation operation, that is an operation testing records ### IVertex -- *Implemented By:* AmazonRdsForPostgreSqlDestination, EventBridgeDestination, GoogleAnalytics4Source, JdbcSmallDataScaleSource, MarketoSource, MicrosoftDynamics365Source, MicrosoftSharepointOnlineSource, RedshiftDestination, S3Destination, S3Source, SAPOdataDestination, SAPOdataSource, SalesforceDestination, SalesforceMarketingCloudSource, SalesforceSource, ServiceNowSource, SlackSource, SnowflakeDestination, ZendeskSource, IDestination, ISource, IVertex +- *Implemented By:* AmazonRdsForPostgreSqlDestination, EventBridgeDestination, GoogleAnalytics4Source, GoogleBigQuerySource, JdbcSmallDataScaleSource, MarketoSource, MicrosoftDynamics365Source, MicrosoftSharepointOnlineSource, RedshiftDestination, S3Destination, S3Source, SAPOdataDestination, SAPOdataSource, SalesforceDestination, SalesforceMarketingCloudSource, SalesforceSource, ServiceNowSource, SlackSource, SnowflakeDestination, ZendeskSource, IDestination, ISource, IVertex An interface representing a vertex, i.e. a source or a destination of an AppFlow flow. @@ -15860,6 +16553,21 @@ The AppFlow type of the connector that this source is implemented for. --- +### GoogleBigQueryApiVersion + +#### Members + +| **Name** | **Description** | +| --- | --- | +| V2 | *No description.* | + +--- + +##### `V2` + +--- + + ### JdbcDriver #### Members diff --git a/package.json b/package.json index 3f5c223b..2b78476a 100644 --- a/package.json +++ b/package.json @@ -122,7 +122,7 @@ "@typescript-eslint/parser": "^6", "aws-cdk": "^2", "aws-cdk-lib": "2.116.1", - "cdklabs-projen-project-types": "^0.1.184", + "cdklabs-projen-project-types": "^0.1.186", "constructs": "10.0.5", "esbuild": "^0.19.11", "eslint": "^8", @@ -135,7 +135,7 @@ "jsii-docgen": "^8.0.56", "jsii-pacmak": "^1.94.0", "jsii-rosetta": "^5.3.3", - "projen": "^0.77.6", + "projen": "^0.79.1", "standard-version": "^9", "ts-jest": "^27", "ts-node": "^10.9.2", diff --git a/src/googlebigquery/profile.ts b/src/googlebigquery/profile.ts index 3f9bb7d7..80309a49 100644 --- a/src/googlebigquery/profile.ts +++ b/src/googlebigquery/profile.ts @@ -2,6 +2,7 @@ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 */ +import { SecretValue } from 'aws-cdk-lib'; import { CfnConnectorProfile } from 'aws-cdk-lib/aws-appflow'; import { Construct } from 'constructs'; import { GoogleBigQueryConnectorType } from './type'; @@ -35,17 +36,17 @@ export interface GoogleBigQueryRefreshTokenGrantFlow { /** * A non-expired refresh token. */ - readonly refreshToken?: string; + readonly refreshToken?: SecretValue; /** * The secret of the client app. */ - readonly clientSecret?: string; + readonly clientSecret?: SecretValue; /** * The id of the client app. */ - readonly clientId?: string; + readonly clientId?: SecretValue; } /** @@ -67,7 +68,7 @@ export interface GoogleBigQueryOAuthSettings { * * @default Retrieves a fresh accessToken with the information in the [flow property]{@link GoogleBigQueryOAuthSettings#flow} */ - readonly accessToken?: string; + readonly accessToken?: SecretValue; /** * The OAuth flow used for obtaining a new accessToken when the old is not present or expired. @@ -124,10 +125,10 @@ export class GoogleBigQueryConnectorProfile extends ConnectorProfileBase { customConnector: { oauth2: { // INFO: when using Refresh Token Grant Flow - access token property is required - accessToken: properties.oAuth.accessToken ?? 'dummyAccessToken', - refreshToken: properties.oAuth.flow?.refreshTokenGrant.refreshToken, - clientId: properties.oAuth.flow?.refreshTokenGrant.clientId, - clientSecret: properties.oAuth.flow?.refreshTokenGrant.clientSecret, + accessToken: properties.oAuth.accessToken?.unsafeUnwrap() ?? 'dummyAccessToken', + refreshToken: properties.oAuth.flow?.refreshTokenGrant.refreshToken?.unsafeUnwrap(), + clientId: properties.oAuth.flow?.refreshTokenGrant.clientId?.unsafeUnwrap(), + clientSecret: properties.oAuth.flow?.refreshTokenGrant.clientSecret?.unsafeUnwrap(), }, authenticationType: ConnectorAuthenticationType.OAUTH2, }, diff --git a/src/index.ts b/src/index.ts index fc225883..d66baad3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ export * from './core'; export * from './amazonrdsforpostgresql'; export * from './eventbridge'; export * from './googleanalytics4'; +export * from './googlebigquery'; export * from './jdbcsmalldatascale'; export * from './marketo'; export * from './microsoftdynamics365'; diff --git a/test/googlebigquery/profile.test.ts b/test/googlebigquery/profile.test.ts new file mode 100644 index 00000000..95efeb7d --- /dev/null +++ b/test/googlebigquery/profile.test.ts @@ -0,0 +1,286 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { SecretValue, Stack } from 'aws-cdk-lib'; +import { Template } from 'aws-cdk-lib/assertions'; +import { Key } from 'aws-cdk-lib/aws-kms'; +import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; + +import { + GoogleBigQueryConnectorProfile, +} from '../../src'; + +describe('GoogleBigQueryConnectorProfile', () => { + + test('OAuth2 profile with direct client credentials exists in the stack', () => { + const stack = new Stack(undefined, 'TestStack', { env: { account: '12345678', region: 'dummy' } }); + + new GoogleBigQueryConnectorProfile(stack, 'TestProfile', { + oAuth: { + accessToken: SecretValue.unsafePlainText('accessToken'), + flow: { + refreshTokenGrant: { + refreshToken: SecretValue.unsafePlainText('refreshToken'), + clientId: SecretValue.unsafePlainText('clientId'), + clientSecret: SecretValue.unsafePlainText('clientSecret'), + }, + }, + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::AppFlow::ConnectorProfile', { + ConnectionMode: 'Public', + ConnectorProfileName: 'TestProfile', + ConnectorType: 'CustomConnector', + ConnectorLabel: 'Google BigQuery', + ConnectorProfileConfig: { + ConnectorProfileCredentials: { + CustomConnector: { + AuthenticationType: 'OAUTH2', + Oauth2: { + AccessToken: 'accessToken', + ClientId: 'clientId', + ClientSecret: 'clientSecret', + RefreshToken: 'refreshToken', + }, + }, + }, + ConnectorProfileProperties: { + CustomConnector: { + OAuth2Properties: { + OAuth2GrantType: 'AUTHORIZATION_CODE', + TokenUrl: 'https://oauth2.googleapis.com/token', + }, + }, + }, + }, + }); + }); + + test('OAuth2 profile with client credentials as secret elements exists in the stack', () => { + const stack = new Stack(undefined, 'TestStack', { env: { account: '12345678', region: 'dummy' } }); + + const secret = new Secret(stack, 'TestSecret'); + + new GoogleBigQueryConnectorProfile(stack, 'TestProfile', { + oAuth: { + accessToken: secret.secretValueFromJson('accessToken'), + flow: { + refreshTokenGrant: { + refreshToken: secret.secretValueFromJson('refreshToken'), + clientId: secret.secretValueFromJson('clientId'), + clientSecret: secret.secretValueFromJson('clientSecret'), + }, + }, + endpoints: { + token: secret.secretValueFromJson('tokenUrl').toString(), + }, + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::AppFlow::ConnectorProfile', { + ConnectionMode: 'Public', + ConnectorProfileName: 'TestProfile', + ConnectorType: 'CustomConnector', + ConnectorLabel: 'Google BigQuery', + ConnectorProfileConfig: { + ConnectorProfileCredentials: { + CustomConnector: { + AuthenticationType: 'OAUTH2', + Oauth2: { + AccessToken: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:accessToken::}}', + ], + ], + }, + ClientId: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:clientId::}}', + ], + ], + }, + ClientSecret: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:clientSecret::}}', + ], + ], + }, + RefreshToken: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:refreshToken::}}', + ], + ], + }, + }, + }, + }, + ConnectorProfileProperties: { + CustomConnector: { + OAuth2Properties: { + OAuth2GrantType: 'AUTHORIZATION_CODE', + TokenUrl: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:tokenUrl::}}', + ], + ], + }, + }, + }, + }, + }, + }); + }); + + + test('OAuth2 profile with a dedicated KMS key and client credentials as secret elements exists in the stack', () => { + const stack = new Stack(undefined, 'TestStack', { env: { account: '12345678', region: 'dummy' } }); + + const key = new Key(stack, 'TestKey'); + + const secret = new Secret(stack, 'TestSecret'); + + new GoogleBigQueryConnectorProfile(stack, 'TestProfile', { + key: key, + oAuth: { + accessToken: secret.secretValueFromJson('accessToken'), + flow: { + refreshTokenGrant: { + refreshToken: secret.secretValueFromJson('refreshToken'), + clientId: secret.secretValueFromJson('clientId'), + clientSecret: secret.secretValueFromJson('clientSecret'), + }, + }, + endpoints: { + token: secret.secretValueFromJson('tokenUrl').toString(), + }, + }, + }); + + Template.fromStack(stack).hasResource('AWS::AppFlow::ConnectorProfile', { + Properties: { + ConnectionMode: 'Public', + ConnectorProfileName: 'TestProfile', + ConnectorType: 'CustomConnector', + ConnectorLabel: 'Google BigQuery', + ConnectorProfileConfig: { + ConnectorProfileCredentials: { + CustomConnector: { + AuthenticationType: 'OAUTH2', + Oauth2: { + AccessToken: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:accessToken::}}', + ], + ], + }, + ClientId: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:clientId::}}', + ], + ], + }, + ClientSecret: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:clientSecret::}}', + ], + ], + }, + RefreshToken: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:refreshToken::}}', + ], + ], + }, + }, + }, + }, + ConnectorProfileProperties: { + CustomConnector: { + OAuth2Properties: { + OAuth2GrantType: 'AUTHORIZATION_CODE', + TokenUrl: { + 'Fn::Join': [ + '', + [ + '{{resolve:secretsmanager:', + { + Ref: 'TestSecret16AF87B1', + }, + ':SecretString:tokenUrl::}}', + ], + ], + }, + }, + }, + }, + }, + KMSArn: { + 'Fn::GetAtt': [ + 'TestKey4CACAF33', + 'Arn', + ], + }, + }, + DependsOn: [ + 'TestKey4CACAF33', + ], + }); + }); + +}); \ No newline at end of file diff --git a/test/googlebigquery/source.test.ts b/test/googlebigquery/source.test.ts new file mode 100644 index 00000000..779bdfee --- /dev/null +++ b/test/googlebigquery/source.test.ts @@ -0,0 +1,224 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +import { SecretValue, Stack } from 'aws-cdk-lib'; +import { Template } from 'aws-cdk-lib/assertions'; +import { Bucket } from 'aws-cdk-lib/aws-s3'; + +import { + Mapping, + GoogleBigQueryApiVersion, + GoogleBigQueryConnectorProfile, + GoogleBigQueryConnectorType, + GoogleBigQuerySource, + OnDemandFlow, + S3Destination, +} from '../../src'; + +describe('GoogleBigQuerySource', () => { + + test('Source with only connector name', () => { + const stack = new Stack(undefined, 'TestStack'); + const source = new GoogleBigQuerySource({ + profile: GoogleBigQueryConnectorProfile.fromConnectionProfileName(stack, 'TestProfile', 'dummy-profile'), + object: { + project: 'projectName', + dataset: 'datasetName', + table: 'tableName', + }, + apiVersion: GoogleBigQueryApiVersion.V2, + }); + + const expectedConnectorType = GoogleBigQueryConnectorType.instance; + expect(source.connectorType.asProfileConnectorLabel).toEqual(expectedConnectorType.asProfileConnectorLabel); + expect(source.connectorType.asProfileConnectorType).toEqual(expectedConnectorType.asProfileConnectorType); + expect(source.connectorType.asTaskConnectorOperatorOrigin).toEqual(expectedConnectorType.asTaskConnectorOperatorOrigin); + expect(source.connectorType.isCustom).toEqual(expectedConnectorType.isCustom); + }); + + test('Source in a Flow is in the stack', () => { + const stack = new Stack(undefined, 'TestStack'); + const source = new GoogleBigQuerySource({ + profile: GoogleBigQueryConnectorProfile.fromConnectionProfileName(stack, 'TestProfile', 'dummy-profile'), + object: { + project: 'projectName', + dataset: 'datasetName', + table: 'tableName', + }, + apiVersion: GoogleBigQueryApiVersion.V2, + }); + + const destination = new S3Destination({ + location: { bucket: new Bucket(stack, 'TestBucket') }, + }); + + new OnDemandFlow(stack, 'TestFlow', { + source: source, + destination: destination, + mappings: [Mapping.mapAll()], + }); + + Template.fromStack(stack).hasResourceProperties('AWS::AppFlow::Flow', { + DestinationFlowConfigList: [ + { + ConnectorType: 'S3', + DestinationConnectorProperties: { + S3: { + BucketName: { + Ref: 'TestBucket560B80BC', + }, + }, + }, + }, + ], + FlowName: 'TestFlow', + SourceFlowConfig: { + ApiVersion: 'v2', + ConnectorProfileName: 'dummy-profile', + ConnectorType: 'CustomConnector', + SourceConnectorProperties: { + CustomConnector: { + EntityName: 'table/projectName/datasetName/tableName', + }, + }, + }, + Tasks: [ + { + ConnectorOperator: { + CustomConnector: 'NO_OP', + }, + SourceFields: [], + TaskProperties: [ + { + Key: 'EXCLUDE_SOURCE_FIELDS_LIST', + Value: '[]', + }, + ], + TaskType: 'Map_all', + }, + ], + TriggerConfig: { + TriggerType: 'OnDemand', + }, + }); + }); + + test('Source for dummy-profile in a Flow is in the stack', () => { + const stack = new Stack(undefined, 'TestStack'); + + const profile = new GoogleBigQueryConnectorProfile(stack, 'TestProfile', { + oAuth: { + accessToken: SecretValue.unsafePlainText('accessToken'), + flow: { + refreshTokenGrant: { + refreshToken: SecretValue.unsafePlainText('refreshToken'), + clientId: SecretValue.unsafePlainText('clientId'), + clientSecret: SecretValue.unsafePlainText('clientSecret'), + }, + }, + }, + }); + + const source = new GoogleBigQuerySource({ + profile: profile, + object: { + project: 'projectName', + dataset: 'datasetName', + table: 'tableName', + }, + apiVersion: GoogleBigQueryApiVersion.V2, + }); + + const destination = new S3Destination({ + location: { bucket: new Bucket(stack, 'TestBucket') }, + }); + + new OnDemandFlow(stack, 'TestFlow', { + source: source, + destination: destination, + mappings: [Mapping.mapAll()], + }); + + const template = Template.fromStack(stack); + + template.hasResourceProperties('AWS::AppFlow::ConnectorProfile', { + ConnectionMode: 'Public', + ConnectorProfileName: 'TestProfile', + ConnectorType: 'CustomConnector', + ConnectorLabel: 'Google BigQuery', + ConnectorProfileConfig: { + ConnectorProfileCredentials: { + CustomConnector: { + AuthenticationType: 'OAUTH2', + Oauth2: { + AccessToken: 'accessToken', + ClientId: 'clientId', + ClientSecret: 'clientSecret', + RefreshToken: 'refreshToken', + }, + }, + }, + ConnectorProfileProperties: { + CustomConnector: { + OAuth2Properties: { + OAuth2GrantType: 'AUTHORIZATION_CODE', + TokenUrl: 'https://oauth2.googleapis.com/token', + }, + }, + }, + }, + }); + + template.hasResource('AWS::AppFlow::Flow', { + Properties: { + DestinationFlowConfigList: [ + { + ConnectorType: 'S3', + DestinationConnectorProperties: { + S3: { + BucketName: { + Ref: 'TestBucket560B80BC', + }, + }, + }, + }, + ], + FlowName: 'TestFlow', + SourceFlowConfig: { + ApiVersion: 'v2', + ConnectorProfileName: 'TestProfile', + ConnectorType: 'CustomConnector', + SourceConnectorProperties: { + CustomConnector: { + EntityName: 'table/projectName/datasetName/tableName', + }, + }, + }, + Tasks: [ + { + ConnectorOperator: { + CustomConnector: 'NO_OP', + }, + SourceFields: [], + TaskProperties: [ + { + Key: 'EXCLUDE_SOURCE_FIELDS_LIST', + Value: '[]', + }, + ], + TaskType: 'Map_all', + }, + ], + TriggerConfig: { + TriggerType: 'OnDemand', + }, + }, + DependsOn: [ + 'TestBucketPolicyBA12ED38', + 'TestBucket560B80BC', + 'TestProfile45C36389', + ], + }); + }); +}); diff --git a/tsconfig.dev.json b/tsconfig.dev.json index 88cc2f2f..da832d0d 100644 --- a/tsconfig.dev.json +++ b/tsconfig.dev.json @@ -26,7 +26,6 @@ "target": "ES2019" }, "include": [ - ".projenrc.js", "src/**/*.ts", "test/**/*.ts", ".projenrc.ts", diff --git a/yarn.lock b/yarn.lock index 5a1bfd4c..6c49c40c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1552,10 +1552,10 @@ case@1.6.3, case@^1.6.3: resolved "https://registry.yarnpkg.com/case/-/case-1.6.3.tgz#0a4386e3e9825351ca2e6216c60467ff5f1ea1c9" integrity sha512-mzDSXIPaFwVDvZAHqZ9VlbyF4yyXRuX6IvB06WvPYkqJVO24kX1PPhv9bfpKNFZyxYFmmgo03HUiD8iklmJYRQ== -cdklabs-projen-project-types@^0.1.184: - version "0.1.184" - resolved "https://registry.yarnpkg.com/cdklabs-projen-project-types/-/cdklabs-projen-project-types-0.1.184.tgz#b5d6fb7ddcb2ffd71d1ecccd723ffe54f0db14a5" - integrity sha512-saF/295s0Pzrk9/N9S7MHDtWWJaK8nYxl68xLQe15sWPitVjCp6hUgEfZh8TWeIpTnXh3N4btZLxSGjUspVS4A== +cdklabs-projen-project-types@^0.1.186: + version "0.1.186" + resolved "https://registry.yarnpkg.com/cdklabs-projen-project-types/-/cdklabs-projen-project-types-0.1.186.tgz#7bf27240000b86d7b495df62abbc393606e96677" + integrity sha512-EF1DMNFFsP+BH/eTB8/Qm8qSd6Sok50CKNrit1tIltJGYOqfJy7CciSrY30NZPDar+/cSgdDv5F2iUs6neW0dw== dependencies: yaml "^2.3.4" @@ -4514,10 +4514,10 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== -projen@^0.77.6: - version "0.77.6" - resolved "https://registry.yarnpkg.com/projen/-/projen-0.77.6.tgz#480c1af4246bba55e8d0732ff7a9d1804ce414be" - integrity sha512-nXbbDr81UjfLjCfVfHGfGPIjiN7INSyMUa52FYupX0TmybMq+CnvX8o0O45feOLLhsifNq7EHXtF+hgBtpBb8A== +projen@^0.79.1: + version "0.79.1" + resolved "https://registry.yarnpkg.com/projen/-/projen-0.79.1.tgz#bff39a8378675dc92d7a880f8f57595b52f51bcd" + integrity sha512-BkkCdGXZ+1pCN6hUZadxclweDfFkH6k1uAUpoY+B4awDDCtQQ7A9/Is6zyc89Jt38xDg7Ctc+/1Z86WwjOIENA== dependencies: "@iarna/toml" "^2.2.5" case "^1.6.3"