From 30b371b568e335548b644656ef3aa19a99327d41 Mon Sep 17 00:00:00 2001 From: Brandon Ly Date: Tue, 27 Feb 2024 14:38:33 -0600 Subject: [PATCH] [HOTFIX]: Add manual trigger for cache updater stack (#1001) * [HOTFIX]: Add manual trigger for cache updater stack * [HOTFIX]: Test * [HOTFIX]: Use correct env * [HOTFIX]: Up CPU and memory limit * [HOTFIX]: Uncomment force run * [HOTFIX]: use main branch for docs-worker-actions * [HOTFIX]: Remove push * [HOTFIX]: Update readme * address comment --- .github/workflows/deploy-prd-enhanced-cache.yml | 6 ++++++ cdk-infra/bin/cdk-infra.ts | 7 ++++++- .../cache-updater-worker-construct.ts | 12 +++++++++--- cdk-infra/lib/stacks/cache-updater-stack.ts | 9 +++++++-- src/cache-updater/README.md | 2 ++ src/cache-updater/index.ts | 16 ++++++++++++++-- 6 files changed, 44 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-prd-enhanced-cache.yml b/.github/workflows/deploy-prd-enhanced-cache.yml index dea4ac949..ef0116e19 100644 --- a/.github/workflows/deploy-prd-enhanced-cache.yml +++ b/.github/workflows/deploy-prd-enhanced-cache.yml @@ -1,6 +1,11 @@ on: release: types: [released] + workflow_dispatch: + inputs: + forceRun: + description: Force the cache to be rebuilt. + default: 'false' concurrency: group: environment-prd-enhanced-cacheUpdate-${{ github.ref }} cancel-in-progress: true @@ -24,3 +29,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WORKSPACE: ${{ github.workspace }} + FORCE_RUN: ${{ inputs.forceRun }} diff --git a/cdk-infra/bin/cdk-infra.ts b/cdk-infra/bin/cdk-infra.ts index 44e4eec80..5a7737137 100644 --- a/cdk-infra/bin/cdk-infra.ts +++ b/cdk-infra/bin/cdk-infra.ts @@ -46,7 +46,12 @@ async function main() { env, }); - new CacheUpdaterStack(app, `${stackName}-cache`, { vpc, env, githubSecret: workerSecureStrings.GITHUB_SECRET }); + new CacheUpdaterStack(app, `${stackName}-cache`, { + vpc, + env, + githubSecret: workerSecureStrings.GITHUB_SECRET, + githubBotPassword: workerSecureStrings.GITHUB_BOT_PASSWORD, + }); } main(); diff --git a/cdk-infra/lib/constructs/cache-updater/cache-updater-worker-construct.ts b/cdk-infra/lib/constructs/cache-updater/cache-updater-worker-construct.ts index 31052e3c2..571ab8e60 100644 --- a/cdk-infra/lib/constructs/cache-updater/cache-updater-worker-construct.ts +++ b/cdk-infra/lib/constructs/cache-updater/cache-updater-worker-construct.ts @@ -6,11 +6,13 @@ import { Bucket } from 'aws-cdk-lib/aws-s3'; import { Construct } from 'constructs'; import path from 'path'; import { getSnootyParserVersion } from '../../../utils/env'; +import { StringParameter } from 'aws-cdk-lib/aws-ssm'; const SNOOTY_CACHE_BUCKET_NAME = 'snooty-parse-cache'; interface CacheUpdaterWorkerConstructProps { vpc: IVpc; + githubBotPassword: string; } export class CacheUpdaterWorkerConstruct extends Construct { @@ -18,7 +20,7 @@ export class CacheUpdaterWorkerConstruct extends Construct { readonly taskDefinition: TaskDefinition; readonly containerName: string; - constructor(scope: Construct, id: string, { vpc }: CacheUpdaterWorkerConstructProps) { + constructor(scope: Construct, id: string, { vpc, githubBotPassword }: CacheUpdaterWorkerConstructProps) { super(scope, id); const cluster = new Cluster(this, 'cacheUpdaterCluster', { @@ -34,8 +36,8 @@ export class CacheUpdaterWorkerConstruct extends Construct { snootyParseCacheBucket.grantWrite(taskRole); const taskDefinition = new FargateTaskDefinition(this, 'cacheUpdaterWorker', { - cpu: 2048, - memoryLimitMiB: 4096, + cpu: 4096, + memoryLimitMiB: 8192, taskRole, }); @@ -43,6 +45,7 @@ export class CacheUpdaterWorkerConstruct extends Construct { const taskDefLogGroup = new LogGroup(this, 'cacheUpdaterWorkerLogGroup'); const snootyParserVersion = getSnootyParserVersion(); + const githubBotUsername = StringParameter.valueFromLookup(this, '/env/prd/docs/worker_pool/github/bot/username'); taskDefinition.addContainer('cacheUpdaterWorkerImage', { image: ContainerImage.fromAsset(path.join(__dirname, '../../../../'), { @@ -52,7 +55,10 @@ export class CacheUpdaterWorkerConstruct extends Construct { }), environment: { SNOOTY_CACHE_BUCKET_NAME, + GITHUB_BOT_PASSWORD: githubBotPassword, + GITHUB_BOT_USERNAME: githubBotUsername, }, + logging: LogDrivers.awsLogs({ streamPrefix: 'cacheupdater', logGroup: taskDefLogGroup, diff --git a/cdk-infra/lib/stacks/cache-updater-stack.ts b/cdk-infra/lib/stacks/cache-updater-stack.ts index a4467a0a7..a87d604d0 100644 --- a/cdk-infra/lib/stacks/cache-updater-stack.ts +++ b/cdk-infra/lib/stacks/cache-updater-stack.ts @@ -7,15 +7,20 @@ import { CacheUpdaterApiConstruct } from '../constructs/cache-updater/cache-upda interface CacheUpdaterStackProps extends StackProps { vpc: Vpc; githubSecret: string; + githubBotPassword: string; } export class CacheUpdaterStack extends Stack { - constructor(scope: Construct, id: string, { vpc, githubSecret, ...props }: CacheUpdaterStackProps) { + constructor( + scope: Construct, + id: string, + { vpc, githubSecret, githubBotPassword, ...props }: CacheUpdaterStackProps + ) { super(scope, id, props); const { clusterName, taskDefinition, containerName } = new CacheUpdaterWorkerConstruct( this, 'cache-updater-resources', - { vpc } + { vpc, githubBotPassword } ); new CacheUpdaterApiConstruct(this, 'cache-updater-api', { diff --git a/src/cache-updater/README.md b/src/cache-updater/README.md index b2b3f8752..3f5d0a205 100644 --- a/src/cache-updater/README.md +++ b/src/cache-updater/README.md @@ -8,6 +8,8 @@ A cache invalidation event occurs under two circumstances: 2. An updated version of the Snooty Parser is used by the Autobuilder; all doc sites' Snooty caches will be rebuilt in this event +The cache update process can be manually triggered by going to the GitHub Actions for the `docs-worker-pool` repository, going to the workflow for the cache updater, and then clicking run workflow. Make sure to set force run to be `true`. + ### Architecture #### Worker diff --git a/src/cache-updater/index.ts b/src/cache-updater/index.ts index ce1d8ac9e..8ee3ddad5 100644 --- a/src/cache-updater/index.ts +++ b/src/cache-updater/index.ts @@ -7,15 +7,27 @@ import { S3Client } from '@aws-sdk/client-s3'; import { executeCliCommand } from '../commands/src/helpers'; const readdirAsync = promisify(fs.readdir); -const SNOOTY_CACHE_BUCKET_NAME = process.env.SNOOTY_CACHE_BUCKET_NAME; +const { SNOOTY_CACHE_BUCKET_NAME, GITHUB_BOT_USERNAME, GITHUB_BOT_PASSWORD } = process.env; if (!SNOOTY_CACHE_BUCKET_NAME) throw new Error('ERROR! SNOOTY_CACHE_BUCKET_NAME is not defined'); async function cloneDocsRepo(repoName: string, repoOwner: string) { + if (!GITHUB_BOT_USERNAME) { + const errorMessage = `ERROR! GITHUB_BOT_USERNAME is not set. ${repoOwner}/${repoName} will not have their cache updated`; + console.error(errorMessage); + throw new Error(errorMessage); + } + + if (!GITHUB_BOT_PASSWORD) { + const errorMessage = `ERROR! GITHUB_BOT_PASSWORD is not set. ${repoOwner}/${repoName} will not have their cache updated`; + console.error(errorMessage); + throw new Error(errorMessage); + } + try { const cloneResults = await executeCliCommand({ command: 'git', - args: ['clone', `https://github.com/${repoOwner}/${repoName}`], + args: ['clone', `https://${GITHUB_BOT_USERNAME}:${GITHUB_BOT_PASSWORD}@github.com/${repoOwner}/${repoName}`], }); console.log('clone: ', cloneResults);