Merge branch 'master' into DOP-4127-no-makefiles

mongodb · Nov 28, 2023 · e352d0b · e352d0b
2 parents c3ae28e + 0be00ff
commit e352d0b
Show file tree

Hide file tree

Showing 17 changed files with 37 additions and 18 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -23,7 +23,7 @@ RUN cd ./modules/oas-page-builder \
 FROM ubuntu:20.04
 ARG WORK_DIRECTORY=/home/docsworker-xlarge
 ARG SNOOTY_PARSER_VERSION=0.15.0
-ARG SNOOTY_FRONTEND_VERSION=0.15.1
+ARG SNOOTY_FRONTEND_VERSION=0.15.1.1b
 ARG MUT_VERSION=0.10.7
 ARG REDOC_CLI_VERSION=1.2.2
 ARG NPM_BASE_64_AUTH

diff --git a/Dockerfile.enhanced b/Dockerfile.enhanced
@@ -23,7 +23,7 @@ RUN cd ./modules/oas-page-builder \
 FROM ubuntu:20.04
 ARG WORK_DIRECTORY=/home/docsworker-xlarge
 ARG SNOOTY_PARSER_VERSION=0.15.0
-ARG SNOOTY_FRONTEND_VERSION=0.15.1.1
+ARG SNOOTY_FRONTEND_VERSION=0.15.1.1b
 ARG MUT_VERSION=0.10.7
 ARG REDOC_CLI_VERSION=1.2.2
 ARG NPM_BASE_64_AUTH

diff --git a/Dockerfile.local b/Dockerfile.local
@@ -2,7 +2,7 @@ FROM arm64v8/ubuntu:20.04
 ARG NPM_BASE_64_AUTH
 ARG NPM_EMAIL
 ARG SNOOTY_PARSER_VERSION=0.15.0
-ARG SNOOTY_FRONTEND_VERSION=0.15.1
+ARG SNOOTY_FRONTEND_VERSION=0.15.1.1b
 ARG MUT_VERSION=0.10.7
 ARG REDOC_CLI_VERSION=1.2.2
 ARG NPM_BASE_64_AUTH

diff --git a/cdk-infra/lib/constructs/worker/worker-env-construct.ts b/cdk-infra/lib/constructs/worker/worker-env-construct.ts
@@ -59,6 +59,7 @@ export class WorkerEnvConstruct extends Construct {
     const repoBranchesCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/repo`);
     const docsetsCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/docsets`);
     const jobCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/job/queue`);
+    const gatsbyMarianUrl = StringParameter.valueFromLookup(this, `${ssmPrefix}/frontend/marian_url`);
 
     const dbPassword = secureStrings['MONGO_ATLAS_PASSWORD'];
     this.environment = {
@@ -93,6 +94,7 @@ export class WorkerEnvConstruct extends Construct {
       GATSBY_TEST_SEARCH_UI: 'false',
       GATSBY_SHOW_CHATBOT: gatsbyUseChatbot,
       GATSBY_HIDE_UNIFIED_FOOTER_LOCALE: gatsbyHideUnifiedFooterLocale,
+      GATSBY_MARIAN_URL: gatsbyMarianUrl,
     };
   }
 }
diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json
@@ -25,6 +25,7 @@
   "featureFlagSearchUI": "GATSBY_TEST_SEARCH_UI",
   "gatsbyUseChatbot": "GATSBY_SHOW_CHATBOT",
   "gatsbyHideUnifiedFooterLocale": "GATSBY_HIDE_UNIFIED_FOOTER_LOCALE",
+  "gatsbyMarianURL": "GATSBY_MARIAN_URL",
   "repoBranchesCollection": "REPO_BRANCHES_COL_NAME",
   "docsetsCollection": "DOCSETS_COL_NAME",
   "repo_dir": "repos",

diff --git a/config/default.json b/config/default.json
@@ -21,6 +21,7 @@
   "PORT": 3000,
   "MAX_STDOUT_BUFFER_SIZE": 16000000,
   "GATSBY_PARSER_USER": "docsworker-xlarge",
+  "gatsbyMarianURL": "https://docs-search-transport.mongodb.com/",
   "gatsbyTestEmbedVersions": false,
   "fastlyOpsManagerToken": "FASTLY_OPS_MANAGER_TOKEN",
   "fastlyOpsManagerServiceId": "FASTLY_OPS_MANAGER_SERVICE_ID",

diff --git a/infrastructure/ecs-main/ecs_service.yml b/infrastructure/ecs-main/ecs_service.yml
@@ -68,6 +68,8 @@ Resources:
               Value: ${self:custom.featureFlagSearchUI}
             - Name: GATSBY_SHOW_CHATBOT
               Value: ${self:custom.gatsbyUseChatbot}
+            - Name: GATSBY_MARIAN_URL
+              Value: ${self:custom.gatsbyMarianURL}
             - Name: GATSBY_HIDE_UNIFIED_FOOTER_LOCALE
               Value: ${self:custom.gatsbyHideUnifiedFooterLocale}
             - Name: FASTLY_MAIN_TOKEN

diff --git a/infrastructure/ecs-main/serverless.yml b/infrastructure/ecs-main/serverless.yml
@@ -120,6 +120,7 @@ custom:
   gatsbyTestEmbedVersions: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/embedded_versions}
   gatsbyUseChatbot: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/use_chatbot}
   gatsbyHideUnifiedFooterLocale: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/hide_locale}
+  gatsbyMarianURL: ${ssm:/env/${self:provider.stage}/docs/worker_pool/frontend/marian_url}
   fastlyMainToken: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/docs/main/token}
   fastlyMainServiceId: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/docs/main/service_id}
   fastlyCloudManagerToken: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/docs/cloudmanager/token}

diff --git a/modules/oas-page-builder/src/services/database.ts b/modules/oas-page-builder/src/services/database.ts
@@ -26,10 +26,12 @@ const getDbName = () => {
   switch (env) {
     // Autobuilder's prd env
     case 'production':
+      return 'snooty_prod';
     case 'dotcomprd':
       return 'snooty_dotcomprd';
     // Autobuilder's pre-prd env
     case 'staging':
+      return 'snooty_stage';
     case 'dotcomstg':
       return 'snooty_dotcomstg';
     default:

diff --git a/modules/oas-page-builder/src/services/pageBuilder.ts b/modules/oas-page-builder/src/services/pageBuilder.ts
@@ -7,8 +7,15 @@ import { findLastSavedVersionData, saveSuccessfulBuildVersionData } from './data
 import { VersionData } from './models/OASFile';
 import { normalizeUrl } from '../utils/normalizeUrl';
 
-const OAS_FILE_SERVER = 'https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/';
-const GIT_HASH_URL = 'https://cloud.mongodb.com/version';
+const env = process.env.SNOOTY_ENV ?? '';
+
+const OAS_FILE_SERVER =
+  env === 'dotcomprd'
+    ? 'https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/'
+    : 'https://mongodb-mms-build-server.s3.amazonaws.com/openapi/';
+
+const GIT_HASH_URL =
+  env === 'dotcomprd' ? 'https://cloud.mongodb.com/version' : 'https://cloud-dev.mongodb.com/version';
 
 const fetchTextData = async (url: string, errMsg: string) => {
   const res = await fetch(url);
@@ -274,7 +281,7 @@ export const buildOpenAPIPages = async (
     if (totalSuccess && sourceType == 'atlas') {
       try {
         const gitHash = await fetchGitHash();
-        const versions = await fetchVersionData(gitHash);
+        const versions = await fetchVersionData(gitHash, OAS_FILE_SERVER);
         await saveSuccessfulBuildVersionData(source, gitHash, versions);
       } catch (e) {
         console.error(e);

diff --git a/modules/oas-page-builder/src/utils/fetchVersionData.ts b/modules/oas-page-builder/src/utils/fetchVersionData.ts
@@ -1,7 +1,7 @@
 import fetch from 'node-fetch';
 
-export const fetchVersionData = async (gitHash: string) => {
-  const versionUrl = `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${gitHash}-api-versions.json`;
+export const fetchVersionData = async (gitHash: string, serverURL: string) => {
+  const versionUrl = `${serverURL}${gitHash}-api-versions.json`;
   const res = await fetch(versionUrl);
   const { versions } = await res.json();
   return versions;

diff --git a/modules/oas-page-builder/tests/unit/services/pageBuilder.test.ts b/modules/oas-page-builder/tests/unit/services/pageBuilder.test.ts
@@ -95,7 +95,7 @@ describe('pageBuilder', () => {
     );
     // Atlas
     expect(mockExecute).toBeCalledWith(
-      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}.json`,
+      `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}.json`,
       getExpectedOutputPath(testOptions.output, testEntries[2][0]),
       expectedAtlasBuildOptions,
       undefined
@@ -139,7 +139,7 @@ describe('pageBuilder', () => {
     );
     // Atlas
     expect(mockExecute).toBeCalledWith(
-      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v1.json`,
+      `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v1.json`,
       getExpectedOutputPath(testOptions.output, testEntries[2][0], API_VERSION),
       expectedAtlasBuildOptions,
       undefined
@@ -211,14 +211,14 @@ describe('pageBuilder', () => {
 
     // Atlas
     expect(mockExecute).toBeCalledWith(
-      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${RESOURCE_VERSION}.json`,
+      `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${RESOURCE_VERSION}.json`,
       getExpectedOutputPath(testOptions.output, testEntries[2][0], API_VERSION, RESOURCE_VERSION),
       expectedAtlasBuildOptions,
       getExpectedVersionOptions(`${SITE_URL}/${testEntries[2][0]}`)
     );
 
     expect(mockExecute).toBeCalledWith(
-      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${RESOURCE_VERSION}.json`,
+      `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${RESOURCE_VERSION}.json`,
       getExpectedOutputPath(testOptions.output, testEntries[2][0], API_VERSION, RESOURCE_VERSION),
       expectedAtlasBuildOptions,
       getExpectedVersionOptions(`${SITE_URL}/${testEntries[2][0]}`)
@@ -254,21 +254,21 @@ describe('pageBuilder', () => {
     await buildOpenAPIPages(testEntries, testOptions);
 
     expect(mockExecute).toBeCalledWith(
-      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${OLDEST_RESOURCE_VERSION}.json`,
+      `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${OLDEST_RESOURCE_VERSION}.json`,
       getExpectedOutputPath(testOptions.output, testEntries[0][0], API_VERSION, OLDEST_RESOURCE_VERSION),
       expectedAtlasBuildOptions,
       getExpectedVersionOptions(`${SITE_URL}/${testEntries[0][0]}`, OLDEST_RESOURCE_VERSION)
     );
 
     expect(mockExecute).toBeCalledWith(
-      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${LATEST_RESOURCE_VERSION}.json`,
+      `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${LATEST_RESOURCE_VERSION}.json`,
       getExpectedOutputPath(testOptions.output, testEntries[0][0], API_VERSION, LATEST_RESOURCE_VERSION),
       expectedAtlasBuildOptions,
       getExpectedVersionOptions(`${SITE_URL}/${testEntries[0][0]}`, LATEST_RESOURCE_VERSION)
     );
 
     expect(mockExecute).toBeCalledWith(
-      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${LATEST_RESOURCE_VERSION}.json`,
+      `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${LATEST_RESOURCE_VERSION}.json`,
       getExpectedOutputPath(testOptions.output, testEntries[0][0], API_VERSION, LATEST_RESOURCE_VERSION),
       expectedAtlasBuildOptions,
       getExpectedVersionOptions(`${SITE_URL}/${testEntries[0][0]}`, LATEST_RESOURCE_VERSION)
@@ -283,7 +283,7 @@ describe('pageBuilder', () => {
 
     await buildOpenAPIPages(testEntries, testOptions);
     expect(mockExecute).toBeCalledWith(
-      `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${LAST_SAVED_GIT_HASH}.json`,
+      `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${LAST_SAVED_GIT_HASH}.json`,
       getExpectedOutputPath(testOptions.output, testEntries[0][0]),
       expectedAtlasBuildOptions,
       undefined

diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts
@@ -36,6 +36,7 @@ async function createEnvProdFile(
       GATSBY_MANIFEST_PATH=${prodDirName}/bundle.zip
       GATSBY_PARSER_USER=${process.env.USER}
       GATSBY_BASE_URL=${baseUrl}
+      GATSBY_MARIAN_URL=${process.env.GATSBY_MARIAN_URL}
       PATH_PREFIX=${prefix}`,
       'utf8'
     );

diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts
@@ -427,6 +427,7 @@ export abstract class JobHandler {
       GATSBY_TEST_SEARCH_UI: this._config.get<string>('featureFlagSearchUI'),
       GATSBY_SHOW_CHATBOT: this._config.get<string>('gatsbyUseChatbot'),
       GATSBY_HIDE_UNIFIED_FOOTER_LOCALE: this._config.get<string>('gatsbyHideUnifiedFooterLocale'),
+      GATSBY_MARIAN_URL: this._config.get<string>('gatsbyMarianURL'),
     };
 
     for (const [envName, envValue] of Object.entries(snootyFrontEndVars)) {

diff --git a/tests/data/data.ts b/tests/data/data.ts
@@ -176,7 +176,7 @@ export class TestDataProvider {
   }
 
   static getEnvVarsWithPathPrefixWithFlags(job: Job): string {
-    return `GATSBY_PARSER_USER=TestUser\nGATSBY_PARSER_BRANCH=${job.payload.branchName}\nPATH_PREFIX=${job.payload.pathPrefix}\nGATSBY_BASE_URL=test\nPREVIEW_BUILD_ENABLED=false\nGATSBY_TEST_SEARCH_UI=false\nGATSBY_SHOW_CHATBOT=false\nGATSBY_HIDE_UNIFIED_FOOTER_LOCALE=true\n`;
+    return `GATSBY_PARSER_USER=TestUser\nGATSBY_PARSER_BRANCH=${job.payload.branchName}\nPATH_PREFIX=${job.payload.pathPrefix}\nGATSBY_BASE_URL=test\nPREVIEW_BUILD_ENABLED=false\nGATSBY_TEST_SEARCH_UI=false\nGATSBY_SHOW_CHATBOT=false\nGATSBY_HIDE_UNIFIED_FOOTER_LOCALE=true\nGATSBY_MARIAN_URL=test-url\n`;
   }
 
   static getPathPrefixCases(): Array<any> {

diff --git a/tests/unit/job/productionJobHandler.test.ts b/tests/unit/job/productionJobHandler.test.ts
@@ -247,7 +247,7 @@ describe('ProductionJobHandler Tests', () => {
 
     expect(jobHandlerTestHelper.fileSystemServices.writeToFile).toBeCalledWith(
       `repos/${jobHandlerTestHelper.job.payload.repoName}/.env.production`,
-      `GATSBY_PARSER_USER=TestUser\nGATSBY_PARSER_BRANCH=${jobHandlerTestHelper.job.payload.branchName}\nPATH_PREFIX=/\nGATSBY_BASE_URL=test\nPREVIEW_BUILD_ENABLED=false\nGATSBY_TEST_SEARCH_UI=false\nGATSBY_SHOW_CHATBOT=false\nGATSBY_HIDE_UNIFIED_FOOTER_LOCALE=true\n`,
+      `GATSBY_PARSER_USER=TestUser\nGATSBY_PARSER_BRANCH=${jobHandlerTestHelper.job.payload.branchName}\nPATH_PREFIX=/\nGATSBY_BASE_URL=test\nPREVIEW_BUILD_ENABLED=false\nGATSBY_TEST_SEARCH_UI=false\nGATSBY_SHOW_CHATBOT=false\nGATSBY_HIDE_UNIFIED_FOOTER_LOCALE=true\nGATSBY_MARIAN_URL=test-url\n`,
       { encoding: 'utf8', flag: 'w' }
     );
   });

diff --git a/tests/utils/jobHandlerTestHelper.ts b/tests/utils/jobHandlerTestHelper.ts
@@ -154,6 +154,7 @@ export class JobHandlerTestHelper {
     this.config.get.calledWith('featureFlagSearchUI').mockReturnValue('false');
     this.config.get.calledWith('gatsbyUseChatbot').mockReturnValue('false');
     this.config.get.calledWith('gatsbyHideUnifiedFooterLocale').mockReturnValue('true');
+    this.config.get.calledWith('gatsbyMarianURL').mockReturnValue('test-url');
     this.repoConnector.checkCommits
       .calledWith(this.job)
       .mockReturnValue(TestDataProvider.getCommitCheckValidResponse(this.job));