diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 0bbf6bb53..72ebe490c 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -46,7 +46,7 @@ jobs: cd cdk-infra/ npm ci npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhook + auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks - name: Update Worker Stack if: steps.filter.outputs.worker == 'true' diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index 19657f31c..e50842d51 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -8,6 +8,8 @@ import { ConsoleLogger } from '../../../src/services/logger'; import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository'; import { EnhancedJob, JobStatus } from '../../../src/entities/job'; import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github'; +import { getMonorepoPaths } from '../../../src/monorepo'; +import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils'; async function prepGithubPushPayload( githubEvent: PushEvent, @@ -75,9 +77,9 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise { + const commitInfo: GitCommitInfo = { + ownerName, + repoName, + commitSha, + }; + + const snootyDirSet = await getSnootyDirSet(commitInfo); + + const projects = updatedFilePaths.map((path) => getProjectDirFromPath(path, snootyDirSet)); + + // remove empty strings and remove duplicated values + return Array.from(new Set(projects.filter((dir) => !!dir))); +} diff --git a/src/monorepo/services/get-paths.ts b/src/monorepo/services/get-paths.ts new file mode 100644 index 000000000..8716d7ec1 --- /dev/null +++ b/src/monorepo/services/get-paths.ts @@ -0,0 +1,36 @@ +import { SNOOTY_TOML_FILENAME } from '../utils/monorepo-constants'; + +/** + * This function returns the project path for a given file change from a docs repository + * within the monorepo. This function supports nested projects. + * @param path An added/modified/removed file path from a commit e.g. server-docs/source/index.rst + * @param commitInfo Contains information + * @returns The closest file path that contains a snooty.toml, relative to the path parameter. + */ +export function getProjectDirFromPath(path: string, snootyDirSet: Set): string { + const pathArray = path.split('/'); + if (pathArray.length === 0) { + console.warn('WARNING! Empty path found: ', path); + return ''; + } + + /** + * If the changed file is the snooty.toml file, we know that we + * are in the project's root directory. We can join the original + * pathArray to get the project path since the snooty.toml has been removed. + */ + const changedFile = pathArray.pop(); + + if (changedFile === SNOOTY_TOML_FILENAME) return pathArray.join('/'); + + while (pathArray.length > 0) { + const currDir = pathArray.join('/'); + + if (snootyDirSet.has(currDir)) return currDir; + + pathArray.pop(); + } + + console.warn(`WARNING! No snooty.toml found for the given path: ${path}`); + return ''; +} diff --git a/src/monorepo/types/atlas-types.ts b/src/monorepo/types/atlas-types.ts new file mode 100644 index 000000000..1af35ee35 --- /dev/null +++ b/src/monorepo/types/atlas-types.ts @@ -0,0 +1,25 @@ +interface DirectoryConfig { + snooty_toml?: string; + source?: string; +} + +interface RepoConfig { + repoName: string; + deployable: boolean; + branches: BranchConfig[]; +} + +interface BranchConfig { + gitBranchName: string; +} + +// TODO: Populate these more. For DOP-3911, they are +// being added for testing purposes. +export interface DocSetEntry { + project: string; + prefix: string; + bucket: string; + url: string; + directories?: DirectoryConfig; + repos?: RepoConfig[]; +} diff --git a/src/monorepo/types/github-types.ts b/src/monorepo/types/github-types.ts new file mode 100644 index 000000000..161613fa8 --- /dev/null +++ b/src/monorepo/types/github-types.ts @@ -0,0 +1,5 @@ +export interface GitCommitInfo { + commitSha: string; + ownerName: string; + repoName: string; +} diff --git a/src/monorepo/utils/monorepo-constants.ts b/src/monorepo/utils/monorepo-constants.ts new file mode 100644 index 000000000..fc71e1bc5 --- /dev/null +++ b/src/monorepo/utils/monorepo-constants.ts @@ -0,0 +1,2 @@ +export const SNOOTY_TOML_FILENAME = 'snooty.toml'; +export const MONOREPO_NAME = 'docs-monorepo'; diff --git a/src/monorepo/utils/path-utils.ts b/src/monorepo/utils/path-utils.ts new file mode 100644 index 000000000..307671e05 --- /dev/null +++ b/src/monorepo/utils/path-utils.ts @@ -0,0 +1,50 @@ +import { Commit } from '@octokit/webhooks-types'; +import { getOctokitClient } from '../../clients/githubClient'; +import { GitCommitInfo } from '../types/github-types'; +import { SNOOTY_TOML_FILENAME } from './monorepo-constants'; + +/** + * Creates a `Set` of all `snooty.toml` paths within the monorepo. + * The function retrieves the monorepo's tree structure from GitHub. + */ +export async function getSnootyDirSet({ commitSha, ownerName, repoName }: GitCommitInfo): Promise> { + try { + const client = getOctokitClient(); + + // getting the repository tree for a given commit SHA. This returns an object + // with the property `tree` that is a flat array of all files in the repository. + // The tree array contains objects that hold the file path. + // Unlike the contents API for repositories, the actual file content is not returned. + const { data } = await client.request('GET /repos/{owner}/{repo}/git/trees/{tree_sha}', { + owner: ownerName, + repo: repoName, + tree_sha: commitSha, + recursive: 'true', + }); + + const snootyTomlDirs = data.tree + .filter((treeNode) => !!treeNode.path?.includes(SNOOTY_TOML_FILENAME)) + .map((treeNode) => { + // casting the `treeNode.path` from `(string | undefined)` to `string` since the filter will ensure that the result + // only includes treeNode.path values that are defined and include snooty.toml + // in the path i.e. we will not have `undefined` as a value in the resulting array. + const path = treeNode.path as string; + + // the - 1 is to remove the trailing slash + return path.slice(0, path.length - SNOOTY_TOML_FILENAME.length - 1); + }); + + const snootyDirSet = new Set(snootyTomlDirs); + + return snootyDirSet; + } catch (error) { + console.error( + `ERROR! Unable to retrieve tree for SHA: ${commitSha} owner name: ${ownerName} repo name: ${repoName}`, + error + ); + throw error; + } +} + +export const getUpdatedFilePaths = (commit: Commit): string[] => + commit.modified.concat(commit.added).concat(commit.removed); diff --git a/src/repositories/docSetRepository.ts b/src/repositories/docSetRepository.ts new file mode 100644 index 000000000..a2a2e25e8 --- /dev/null +++ b/src/repositories/docSetRepository.ts @@ -0,0 +1,45 @@ +import { IConfig } from 'config'; +import { Db } from 'mongodb'; +import { ILogger } from '../services/logger'; +import { BaseRepository } from './baseRepository'; + +const docSetCollectionName = process.env.DOCS_SET_COLLECTION_NAME || 'docset'; + +export class DocSetRepository extends BaseRepository { + constructor(db: Db, config: IConfig, logger: ILogger) { + super(config, logger, 'DocSetRepository', db.collection(docSetCollectionName)); + } + + /** + * Compares the project path from a monorepo push event, and compares it with + * what is configured in the docset entry in Atlas. + * @param path The project path where the snooty.toml file exists from the monorepo. + * This path will reflect the current project path from a given commit. + * @param projectName The project name for the docset entry. + * @returns A boolean representing whether or not the configured docset entry snooty_toml path + * matches the path found in GitHub. + */ + async checkSnootyTomlPath(path: string, projectName: string) { + const query = { project: projectName }; + try { + const docSetObject = await this.findOne( + query, + `Mongo Timeout Error: Timedout while retrieving repos entry for ${path}` + ); + + if (!docSetObject) { + console.warn(`WARNING: The docset does not exist for the following project: ${projectName} \n path: ${path}`); + + return false; + } + + return docSetObject.directories.snooty_toml === path; + } catch (error) { + console.warn( + `WARNING: Error occurred when retrieving project path for ${projectName}. The following path was provided: ${path}`, + error + ); + return false; + } + } +} diff --git a/tests/unit/monorepo/monorepo.test.ts b/tests/unit/monorepo/monorepo.test.ts new file mode 100644 index 000000000..918d48e40 --- /dev/null +++ b/tests/unit/monorepo/monorepo.test.ts @@ -0,0 +1,98 @@ +import { Octokit } from '@octokit/rest'; +import { getMonorepoPaths } from '../../../src/monorepo'; +import { getOctokitClient } from '../../../src/clients/githubClient'; +import { mockDeep } from 'jest-mock-extended'; + +jest.mock('../../../src/clients/githubClient'); +jest.mock('@octokit/rest'); + +const mockedOctokit = mockDeep(); + +beforeEach(() => { + jest.resetAllMocks(); + + const mockedGetOctokitClient = getOctokitClient as jest.MockedFunction; + mockedGetOctokitClient.mockReturnValue(mockedOctokit); +}); + +function mockOctokitTreeResponse(filePaths: string[]) { + // Partial representation of the GitHub API response that we care about. + // The response contains a property `tree` which is an array of objects. + const mockedResponse = { + data: { + tree: filePaths.map((path) => ({ path })), + }, + }; + + jest + .spyOn(mockedOctokit, 'request') + .mockResolvedValueOnce(mockedResponse as unknown as ReturnType); +} + +describe('Monorepo Path Parsing tests', () => { + it('Successfully finds project paths if snooty.toml is changed', async () => { + mockOctokitTreeResponse(['server-docs/source/datalake/snooty.toml', 'server-docs/snooty.toml']); + + const paths = await getMonorepoPaths({ + commitSha: '12345', + ownerName: 'mongodb', + repoName: 'monorepo', + updatedFilePaths: ['server-docs/snooty.toml', 'server-docs/source/datalake/snooty.toml'], + }); + + expect(paths).toContain('server-docs'); + expect(paths).toContain('server-docs/source/datalake'); + }); + + it('Successfully finds project paths based on changed files other than snooty.toml', async () => { + /** + * server-docs/source/datalake contains a snooty.toml file. We will reject once and then resolve + * once as this should mimic responses from the GitHub API. + */ + + mockOctokitTreeResponse(['server-docs/source/datalake/snooty.toml', 'server-docs/snooty.toml']); + + const paths = await getMonorepoPaths({ + commitSha: '12345', + ownerName: 'mongodb', + repoName: 'monorepo', + updatedFilePaths: ['server-docs/source/datalake/source/index.rst'], + }); + + expect(paths).toContain('server-docs/source/datalake'); + }); + + it('Returns an empty array if there is no snooty.toml at any point in the file path', async () => { + mockOctokitTreeResponse(['server-docs/source/datalake/snooty.toml', 'server-docs/snooty.toml']); + + const paths = await getMonorepoPaths({ + commitSha: '12345', + ownerName: 'mongodb', + repoName: 'monorepo', + updatedFilePaths: ['bad/path/index.rst'], + }); + + expect(paths.length).toEqual(0); + }); + + it('Returns only one project path when two files in the same project are modified', async () => { + /** + * server-docs/source/datalake contains a snooty.toml file. We will reject once and then resolve + * once as this should mimic responses from the GitHub API. + */ + mockOctokitTreeResponse(['server-docs/source/datalake/snooty.toml', 'server-docs/snooty.toml']); + + const paths = await getMonorepoPaths({ + commitSha: '12345', + ownerName: 'mongodb', + repoName: 'monorepo', + updatedFilePaths: [ + 'server-docs/source/datalake/source/index.rst', + 'server-docs/source/datalake/source/test/index.rst', + ], + }); + + expect(paths).toContain('server-docs/source/datalake'); + expect(paths.length).toEqual(1); + }); +});