From d0e55afdfbd579891545f22227f1481b56f63fcb Mon Sep 17 00:00:00 2001 From: Cory Hall <43035978+corymhall@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:44:03 -0500 Subject: [PATCH] feat: configure additional job settings for docker asset publishing jobs (#846) Currently if you want to change any of the job settings for the docker asset publishing jobs you can use escape hatches. This PR adds a new property `dockerAssetJobSettings` which allow you to provide additional settings when you create the workflow so you don't have to use escape hatches. Currently I've only implemented support for two things: 1. `setupSteps` - These are additional steps to run just prior to building/publishing the image. The use case I've run into for this is the need to setup docker `buildx` prior to running the build. 2. `permissions` - Allow setting/overriding permissions for docker asset steps. For example, when interacting with GitHub packages you have to set the `packages: read` permission. Fixes # --- API.md | 110 +++++++++++++++++++++++++++++++++++++ README.md | 49 +++++++++++++++++ rosetta/default.ts-fixture | 3 +- src/pipeline.ts | 55 ++++++++++++++++--- test/docker.test.ts | 41 +++++++++++++- yarn.lock | 16 +++--- 6 files changed, 256 insertions(+), 18 deletions(-) diff --git a/API.md b/API.md index 4e7b8c78..e2a9c317 100644 --- a/API.md +++ b/API.md @@ -43,6 +43,7 @@ Workflows. - [Waves for Parallel Builds](#waves-for-parallel-builds) - [Manual Approval Step](#manual-approval-step) - [Pipeline YAML Comments](#pipeline-yaml-comments) + - [Common Configuration for Docker Asset Publishing Steps](#common-configuration-for-docker-asset-publishing) - [Tutorial](#tutorial) - [Not supported yet](#not-supported-yet) - [Contributing](#contributing) @@ -592,6 +593,54 @@ on: < the rest of the pipeline YAML contents> ``` +### Common Configuration for Docker Asset Publishing Steps + +You can provide common job configuration for all of the docker asset publishing +jobs using the `dockerAssetJobSettings` property. You can use this to: + +- Set additional `permissions` at the job level +- Run additional steps prior to the docker build/push step + +Below is an example of example of configuration an additional `permission` which +allows the job to authenticate against GitHub packages. It also shows +configuration additional `setupSteps`, in this case setup steps to configure +docker `buildx` and `QEMU` to enable building images for arm64 architecture. + +```ts +import { ShellStep } from 'aws-cdk-lib/pipelines'; + +const app = new App(); + +const pipeline = new GitHubWorkflow(app, 'Pipeline', { + synth: new ShellStep('Build', { + commands: [ + 'yarn install', + 'yarn build', + ], + }), + dockerAssetJobSettings: { + permissions: { + packages: JobPermission.READ, + }, + setupSteps: [ + { + name: 'Setup Docker QEMU', + uses: 'docker/setup-qemu-action@v3', + }, + { + name: 'Setup Docker buildx', + uses: 'docker/setup-buildx-action@v3', + }, + ], + }, + awsCreds: AwsCredentials.fromOpenIdConnect({ + gitHubActionRoleArn: 'arn:aws:iam:::role/GitHubActionRole', + }), +}); + +app.synth(); +``` + ## Tutorial You can find an example usage in [test/example-app.ts](./test/example-app.ts) @@ -1859,6 +1908,53 @@ const deploymentStatusOptions: DeploymentStatusOptions = { ... } ``` +### DockerAssetJobSettings + +Job level settings applied to all docker asset publishing jobs in the workflow. + +#### Initializer + +```typescript +import { DockerAssetJobSettings } from 'cdk-pipelines-github' + +const dockerAssetJobSettings: DockerAssetJobSettings = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| permissions | JobPermissions | Additional permissions to grant to the docker image publishing job. | +| setupSteps | JobStep[] | GitHub workflow steps to execute before building and publishing the image. | + +--- + +##### `permissions`Optional + +```typescript +public readonly permissions: JobPermissions; +``` + +- *Type:* JobPermissions +- *Default:* no additional permissions + +Additional permissions to grant to the docker image publishing job. + +--- + +##### `setupSteps`Optional + +```typescript +public readonly setupSteps: JobStep[]; +``` + +- *Type:* JobStep[] +- *Default:* [] + +GitHub workflow steps to execute before building and publishing the image. + +--- + ### DockerHubCredentialSecrets Locations of GitHub Secrets used to authenticate to DockerHub. @@ -2451,6 +2547,7 @@ const gitHubWorkflowProps: GitHubWorkflowProps = { ... } | awsCreds | AwsCredentialsProvider | Configure provider for AWS credentials used for deployment. | | buildContainer | ContainerOptions | Build container options. | | cdkCliVersion | string | Version of the CDK CLI to use. | +| dockerAssetJobSettings | DockerAssetJobSettings | Job level settings applied to all docker asset publishing jobs in the workflow. | | dockerCredentials | DockerCredential[] | The Docker Credentials to use to login. | | gitHubActionRoleArn | string | A role that utilizes the GitHub OIDC Identity Provider in your AWS account. | | jobSettings | JobSettings | Job level settings that will be applied to all jobs in the workflow, including synth and asset deploy jobs. | @@ -2537,6 +2634,19 @@ Version of the CDK CLI to use. --- +##### `dockerAssetJobSettings`Optional + +```typescript +public readonly dockerAssetJobSettings: DockerAssetJobSettings; +``` + +- *Type:* DockerAssetJobSettings +- *Default:* no additional settings + +Job level settings applied to all docker asset publishing jobs in the workflow. + +--- + ##### `dockerCredentials`Optional ```typescript diff --git a/README.md b/README.md index c8980cb5..93ff971c 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ Workflows. - [Waves for Parallel Builds](#waves-for-parallel-builds) - [Manual Approval Step](#manual-approval-step) - [Pipeline YAML Comments](#pipeline-yaml-comments) + - [Common Configuration for Docker Asset Publishing Steps](#common-configuration-for-docker-asset-publishing) - [Tutorial](#tutorial) - [Not supported yet](#not-supported-yet) - [Contributing](#contributing) @@ -592,6 +593,54 @@ on: < the rest of the pipeline YAML contents> ``` +### Common Configuration for Docker Asset Publishing Steps + +You can provide common job configuration for all of the docker asset publishing +jobs using the `dockerAssetJobSettings` property. You can use this to: + +- Set additional `permissions` at the job level +- Run additional steps prior to the docker build/push step + +Below is an example of example of configuration an additional `permission` which +allows the job to authenticate against GitHub packages. It also shows +configuration additional `setupSteps`, in this case setup steps to configure +docker `buildx` and `QEMU` to enable building images for arm64 architecture. + +```ts +import { ShellStep } from 'aws-cdk-lib/pipelines'; + +const app = new App(); + +const pipeline = new GitHubWorkflow(app, 'Pipeline', { + synth: new ShellStep('Build', { + commands: [ + 'yarn install', + 'yarn build', + ], + }), + dockerAssetJobSettings: { + permissions: { + packages: JobPermission.READ, + }, + setupSteps: [ + { + name: 'Setup Docker QEMU', + uses: 'docker/setup-qemu-action@v3', + }, + { + name: 'Setup Docker buildx', + uses: 'docker/setup-buildx-action@v3', + }, + ], + }, + awsCreds: AwsCredentials.fromOpenIdConnect({ + gitHubActionRoleArn: 'arn:aws:iam:::role/GitHubActionRole', + }), +}); + +app.synth(); +``` + ## Tutorial You can find an example usage in [test/example-app.ts](./test/example-app.ts) diff --git a/rosetta/default.ts-fixture b/rosetta/default.ts-fixture index 765f0e70..26d3b0e7 100644 --- a/rosetta/default.ts-fixture +++ b/rosetta/default.ts-fixture @@ -15,6 +15,7 @@ import { JsonPatch, Runner, DockerCredential, + JobPermission, } from 'cdk-pipelines-github'; const BETA_ENV = { @@ -34,4 +35,4 @@ class Fixture extends Stack { /// here } -} \ No newline at end of file +} diff --git a/src/pipeline.ts b/src/pipeline.ts index ab8138f1..c2a2c613 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -18,6 +18,25 @@ import { YamlFile } from './yaml-file'; const CDKOUT_ARTIFACT = 'cdk.out'; const ASSET_HASH_NAME = 'asset-hash'; +/** + * Job level settings applied to all docker asset publishing jobs in the workflow. + */ +export interface DockerAssetJobSettings { + /** + * GitHub workflow steps to execute before building and publishing the image. + * + * @default [] + */ + readonly setupSteps?: github.JobStep[]; + + /** + * Additional permissions to grant to the docker image publishing job. + * + * @default - no additional permissions + */ + readonly permissions?: github.JobPermissions; +} + /** * Job level settings applied to all jobs in the workflow. */ @@ -155,6 +174,13 @@ export interface GitHubWorkflowProps extends PipelineBaseProps { * @see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-only-run-job-for-specific-repository */ readonly jobSettings?: JobSettings; + + /** + * Job level settings applied to all docker asset publishing jobs in the workflow. + * + * @default - no additional settings + */ + readonly dockerAssetJobSettings?: DockerAssetJobSettings; } /** @@ -186,6 +212,7 @@ export class GitHubWorkflow extends PipelineBase { } > = {}; private readonly jobSettings?: JobSettings; + private readonly dockerAssetJobSettings?: DockerAssetJobSettings; // in order to keep track of if this pipeline has been built so we can // catch later calls to addWave() or addStage() private builtGH = false; @@ -199,6 +226,7 @@ export class GitHubWorkflow extends PipelineBase { this.preBuildSteps = props.preBuildSteps ?? []; this.postBuildSteps = props.postBuildSteps ?? []; this.jobSettings = props.jobSettings; + this.dockerAssetJobSettings = props.dockerAssetJobSettings; this.awsCredentials = this.getAwsCredentials(props); @@ -492,13 +520,28 @@ export class GitHubWorkflow extends PipelineBase { const cdkoutDir = options.assemblyDir; const jobId = node.uniqueId; const assetId = assets[0].assetId; + const preBuildSteps: github.JobStep[] = []; + let permissions: github.JobPermissions = { + contents: github.JobPermission.READ, + idToken: this.awsCredentials.jobPermission(), + }; // check if asset is docker asset and if we have docker credentials const dockerLoginSteps: github.JobStep[] = []; - if (node.uniqueId.includes('DockerAsset') && this.dockerCredentials.length > 0) { - for (const creds of this.dockerCredentials) { - dockerLoginSteps.push(...this.stepsToConfigureDocker(creds)); + if (node.uniqueId.includes('DockerAsset')) { + if (this.dockerCredentials.length > 0) { + for (const creds of this.dockerCredentials) { + dockerLoginSteps.push(...this.stepsToConfigureDocker(creds)); + } + } + if (this.dockerAssetJobSettings?.setupSteps) { + preBuildSteps.push(...this.dockerAssetJobSettings.setupSteps); } + + permissions = { + ...permissions, + ...this.dockerAssetJobSettings?.permissions, + }; } // create one file and make one step @@ -527,10 +570,7 @@ export class GitHubWorkflow extends PipelineBase { name: `Publish Assets ${jobId}`, ...this.renderJobSettingParameters(), needs: this.renderDependencies(node), - permissions: { - contents: github.JobPermission.READ, - idToken: this.awsCredentials.jobPermission(), - }, + permissions, runsOn: this.runner.runsOn, outputs: { [ASSET_HASH_NAME]: `\${{ steps.Publish.outputs.${ASSET_HASH_NAME} }}`, @@ -543,6 +583,7 @@ export class GitHubWorkflow extends PipelineBase { }, ...this.stepsToConfigureAws(this.publishAssetsAuthRegion), ...dockerLoginSteps, + ...preBuildSteps, publishStep, ], }, diff --git a/test/docker.test.ts b/test/docker.test.ts index 22c9cf93..7b22e095 100644 --- a/test/docker.test.ts +++ b/test/docker.test.ts @@ -6,7 +6,7 @@ import * as codebuild from 'aws-cdk-lib/aws-codebuild'; import { ShellStep } from 'aws-cdk-lib/pipelines'; import * as YAML from 'yaml'; import { TestApp } from './testutil'; -import { DockerCredential, GitHubWorkflow } from '../src'; +import { DockerAssetJobSettings, DockerCredential, GitHubWorkflow, JobPermission } from '../src'; const dockers = join(__dirname, 'demo-image'); @@ -106,9 +106,45 @@ describe('correct format for docker credentials:', () => { }, }); }); + + test('with setup job steps', () => { + const github = createDockerGithubWorkflow(app, [DockerCredential.dockerHub()], { + setupSteps: [ + { + name: 'Setup Docker buildx', + uses: 'docker/setup-buildx-action@v3', + }, + ], + }); + const file = fs.readFileSync(github.workflowPath, 'utf-8'); + const workflow = YAML.parse(file); + const steps = findStepByJobAndUses(workflow, 'Assets-DockerAsset1', 'docker/setup-buildx-action@v3'); + expect(steps.length).toEqual(1); + expect(steps[0]).toEqual({ + name: 'Setup Docker buildx', + uses: 'docker/setup-buildx-action@v3', + }); + }); + + test('with permissions', () => { + const github = createDockerGithubWorkflow(app, [DockerCredential.dockerHub()], { + permissions: { + packages: JobPermission.READ, + }, + }); + const file = fs.readFileSync(github.workflowPath, 'utf-8'); + const workflow = YAML.parse(file); + + const permissions = workflow.jobs['Assets-DockerAsset1'].permissions; + expect(permissions).toEqual({ + 'contents': 'read', + 'id-token': 'none', + 'packages': 'read', + }); + }); }); -function createDockerGithubWorkflow(app: App, dockerCredentials: DockerCredential[]) { +function createDockerGithubWorkflow(app: App, dockerCredentials: DockerCredential[], dockerAssetJobSettings?: DockerAssetJobSettings) { const github = new GitHubWorkflow(app, 'Pipeline', { workflowPath: `${mkoutdir()}/.github/workflows/deploy.yml`, synth: new ShellStep('Build', { @@ -116,6 +152,7 @@ function createDockerGithubWorkflow(app: App, dockerCredentials: DockerCredentia commands: ['yarn build'], }), dockerCredentials, + dockerAssetJobSettings, }); github.addStage(new MyDockerStage(app, 'MyStage', { diff --git a/yarn.lock b/yarn.lock index 4f2b4d18..efa34b31 100644 --- a/yarn.lock +++ b/yarn.lock @@ -311,7 +311,7 @@ debug "^4.3.1" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.6", "@babel/types@^7.3.3": +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.6", "@babel/types@^7.3.0", "@babel/types@^7.3.3": version "7.23.6" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.6.tgz#be33fdb151e1f5a56877d704492c240fc71c7ccd" integrity sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg== @@ -767,11 +767,11 @@ "@babel/types" "^7.0.0" "@types/babel__traverse@*", "@types/babel__traverse@^7.0.4", "@types/babel__traverse@^7.0.6": - version "7.20.5" - resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.5.tgz#7b7502be0aa80cc4ef22978846b983edaafcd4dd" - integrity sha512-WXCyOcRtH37HAUkpXhUduaxdm82b4GSlyTqajXviN4EfiuPgNYR109xMCKvpl6zPIpua0DGlMEDCq+g8EdoheQ== + version "7.18.2" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.18.2.tgz#235bf339d17185bdec25e024ca19cce257cc7309" + integrity sha512-FcFaxOr2V5KZCviw1TnutEMVUVsGt4D2hP1TAfXZAMKuHYW3xQhe3jTxNPWutgCJ3/X1c5yX8ZoGVEItxKbwBg== dependencies: - "@babel/types" "^7.20.7" + "@babel/types" "^7.3.0" "@types/glob@*": version "8.1.0" @@ -855,9 +855,9 @@ integrity sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA== "@types/prettier@^2.1.5": - version "2.7.3" - resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.3.tgz#3e51a17e291d01d17d3fc61422015a933af7a08f" - integrity sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA== + version "2.6.0" + resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.6.0.tgz#efcbd41937f9ae7434c714ab698604822d890759" + integrity sha512-G/AdOadiZhnJp0jXCaBQU449W2h716OW/EoXeYkCytxKL06X1WCXB4DZpp8TpZ8eyIJVS1cw4lrlkkSYU21cDw== "@types/semver@^7.5.0": version "7.5.6"