diff --git a/Dockerfile b/Dockerfile index 7066c11c6..5e0decb75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN cd ./modules/oas-page-builder \ FROM ubuntu:20.04 ARG WORK_DIRECTORY=/home/docsworker-xlarge ARG SNOOTY_PARSER_VERSION=0.15.0 -ARG SNOOTY_FRONTEND_VERSION=0.15.1 +ARG SNOOTY_FRONTEND_VERSION=0.15.1.1b ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.2 ARG NPM_BASE_64_AUTH diff --git a/Dockerfile.enhanced b/Dockerfile.enhanced index 80f782cc5..446e7beff 100644 --- a/Dockerfile.enhanced +++ b/Dockerfile.enhanced @@ -23,7 +23,7 @@ RUN cd ./modules/oas-page-builder \ FROM ubuntu:20.04 ARG WORK_DIRECTORY=/home/docsworker-xlarge ARG SNOOTY_PARSER_VERSION=0.15.0 -ARG SNOOTY_FRONTEND_VERSION=0.15.1.1 +ARG SNOOTY_FRONTEND_VERSION=0.15.1.1b ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.2 ARG NPM_BASE_64_AUTH diff --git a/Dockerfile.local b/Dockerfile.local index 2d1a2ad72..cb542cdd2 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -2,7 +2,7 @@ FROM arm64v8/ubuntu:20.04 ARG NPM_BASE_64_AUTH ARG NPM_EMAIL ARG SNOOTY_PARSER_VERSION=0.15.0 -ARG SNOOTY_FRONTEND_VERSION=0.15.1 +ARG SNOOTY_FRONTEND_VERSION=0.15.1.1b ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.2 ARG NPM_BASE_64_AUTH diff --git a/cdk-infra/lib/constructs/worker/worker-env-construct.ts b/cdk-infra/lib/constructs/worker/worker-env-construct.ts index ed39f0de8..48a7589c8 100644 --- a/cdk-infra/lib/constructs/worker/worker-env-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-env-construct.ts @@ -59,6 +59,7 @@ export class WorkerEnvConstruct extends Construct { const repoBranchesCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/repo`); const docsetsCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/docsets`); const jobCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/job/queue`); + const gatsbyMarianUrl = StringParameter.valueFromLookup(this, `${ssmPrefix}/frontend/marian_url`); const dbPassword = secureStrings['MONGO_ATLAS_PASSWORD']; this.environment = { @@ -93,6 +94,7 @@ export class WorkerEnvConstruct extends Construct { GATSBY_TEST_SEARCH_UI: 'false', GATSBY_SHOW_CHATBOT: gatsbyUseChatbot, GATSBY_HIDE_UNIFIED_FOOTER_LOCALE: gatsbyHideUnifiedFooterLocale, + GATSBY_MARIAN_URL: gatsbyMarianUrl, }; } } diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index be28f8a79..6093f03ee 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -25,6 +25,7 @@ "featureFlagSearchUI": "GATSBY_TEST_SEARCH_UI", "gatsbyUseChatbot": "GATSBY_SHOW_CHATBOT", "gatsbyHideUnifiedFooterLocale": "GATSBY_HIDE_UNIFIED_FOOTER_LOCALE", + "gatsbyMarianURL": "GATSBY_MARIAN_URL", "repoBranchesCollection": "REPO_BRANCHES_COL_NAME", "docsetsCollection": "DOCSETS_COL_NAME", "repo_dir": "repos", diff --git a/config/default.json b/config/default.json index 9b0a73341..db824094f 100644 --- a/config/default.json +++ b/config/default.json @@ -21,6 +21,7 @@ "PORT": 3000, "MAX_STDOUT_BUFFER_SIZE": 16000000, "GATSBY_PARSER_USER": "docsworker-xlarge", + "gatsbyMarianURL": "https://docs-search-transport.mongodb.com/", "gatsbyTestEmbedVersions": false, "fastlyOpsManagerToken": "FASTLY_OPS_MANAGER_TOKEN", "fastlyOpsManagerServiceId": "FASTLY_OPS_MANAGER_SERVICE_ID", diff --git a/infrastructure/ecs-main/ecs_service.yml b/infrastructure/ecs-main/ecs_service.yml index e5799746f..2fdad6e1d 100644 --- a/infrastructure/ecs-main/ecs_service.yml +++ b/infrastructure/ecs-main/ecs_service.yml @@ -68,6 +68,8 @@ Resources: Value: ${self:custom.featureFlagSearchUI} - Name: GATSBY_SHOW_CHATBOT Value: ${self:custom.gatsbyUseChatbot} + - Name: GATSBY_MARIAN_URL + Value: ${self:custom.gatsbyMarianURL} - Name: GATSBY_HIDE_UNIFIED_FOOTER_LOCALE Value: ${self:custom.gatsbyHideUnifiedFooterLocale} - Name: FASTLY_MAIN_TOKEN diff --git a/infrastructure/ecs-main/serverless.yml b/infrastructure/ecs-main/serverless.yml index c08bc710a..603febff7 100644 --- a/infrastructure/ecs-main/serverless.yml +++ b/infrastructure/ecs-main/serverless.yml @@ -120,6 +120,7 @@ custom: gatsbyTestEmbedVersions: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/embedded_versions} gatsbyUseChatbot: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/use_chatbot} gatsbyHideUnifiedFooterLocale: ${ssm:/env/${self:provider.stage}/docs/worker_pool/flag/hide_locale} + gatsbyMarianURL: ${ssm:/env/${self:provider.stage}/docs/worker_pool/frontend/marian_url} fastlyMainToken: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/docs/main/token} fastlyMainServiceId: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/docs/main/service_id} fastlyCloudManagerToken: ${ssm:/env/${self:provider.stage}/docs/worker_pool/fastly/docs/cloudmanager/token} diff --git a/modules/oas-page-builder/src/services/database.ts b/modules/oas-page-builder/src/services/database.ts index bab7dc988..22515fb82 100644 --- a/modules/oas-page-builder/src/services/database.ts +++ b/modules/oas-page-builder/src/services/database.ts @@ -26,10 +26,12 @@ const getDbName = () => { switch (env) { // Autobuilder's prd env case 'production': + return 'snooty_prod'; case 'dotcomprd': return 'snooty_dotcomprd'; // Autobuilder's pre-prd env case 'staging': + return 'snooty_stage'; case 'dotcomstg': return 'snooty_dotcomstg'; default: diff --git a/modules/oas-page-builder/src/services/pageBuilder.ts b/modules/oas-page-builder/src/services/pageBuilder.ts index a25379be5..9b41bf050 100644 --- a/modules/oas-page-builder/src/services/pageBuilder.ts +++ b/modules/oas-page-builder/src/services/pageBuilder.ts @@ -7,8 +7,15 @@ import { findLastSavedVersionData, saveSuccessfulBuildVersionData } from './data import { VersionData } from './models/OASFile'; import { normalizeUrl } from '../utils/normalizeUrl'; -const OAS_FILE_SERVER = 'https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/'; -const GIT_HASH_URL = 'https://cloud.mongodb.com/version'; +const env = process.env.SNOOTY_ENV ?? ''; + +const OAS_FILE_SERVER = + env === 'dotcomprd' + ? 'https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/' + : 'https://mongodb-mms-build-server.s3.amazonaws.com/openapi/'; + +const GIT_HASH_URL = + env === 'dotcomprd' ? 'https://cloud.mongodb.com/version' : 'https://cloud-dev.mongodb.com/version'; const fetchTextData = async (url: string, errMsg: string) => { const res = await fetch(url); @@ -274,7 +281,7 @@ export const buildOpenAPIPages = async ( if (totalSuccess && sourceType == 'atlas') { try { const gitHash = await fetchGitHash(); - const versions = await fetchVersionData(gitHash); + const versions = await fetchVersionData(gitHash, OAS_FILE_SERVER); await saveSuccessfulBuildVersionData(source, gitHash, versions); } catch (e) { console.error(e); diff --git a/modules/oas-page-builder/src/utils/fetchVersionData.ts b/modules/oas-page-builder/src/utils/fetchVersionData.ts index 5090b133c..953f6d20f 100644 --- a/modules/oas-page-builder/src/utils/fetchVersionData.ts +++ b/modules/oas-page-builder/src/utils/fetchVersionData.ts @@ -1,7 +1,7 @@ import fetch from 'node-fetch'; -export const fetchVersionData = async (gitHash: string) => { - const versionUrl = `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${gitHash}-api-versions.json`; +export const fetchVersionData = async (gitHash: string, serverURL: string) => { + const versionUrl = `${serverURL}${gitHash}-api-versions.json`; const res = await fetch(versionUrl); const { versions } = await res.json(); return versions; diff --git a/modules/oas-page-builder/tests/unit/services/pageBuilder.test.ts b/modules/oas-page-builder/tests/unit/services/pageBuilder.test.ts index dbe57a3af..5d01f1bc8 100644 --- a/modules/oas-page-builder/tests/unit/services/pageBuilder.test.ts +++ b/modules/oas-page-builder/tests/unit/services/pageBuilder.test.ts @@ -95,7 +95,7 @@ describe('pageBuilder', () => { ); // Atlas expect(mockExecute).toBeCalledWith( - `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}.json`, + `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}.json`, getExpectedOutputPath(testOptions.output, testEntries[2][0]), expectedAtlasBuildOptions, undefined @@ -139,7 +139,7 @@ describe('pageBuilder', () => { ); // Atlas expect(mockExecute).toBeCalledWith( - `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v1.json`, + `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v1.json`, getExpectedOutputPath(testOptions.output, testEntries[2][0], API_VERSION), expectedAtlasBuildOptions, undefined @@ -211,14 +211,14 @@ describe('pageBuilder', () => { // Atlas expect(mockExecute).toBeCalledWith( - `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${RESOURCE_VERSION}.json`, + `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${RESOURCE_VERSION}.json`, getExpectedOutputPath(testOptions.output, testEntries[2][0], API_VERSION, RESOURCE_VERSION), expectedAtlasBuildOptions, getExpectedVersionOptions(`${SITE_URL}/${testEntries[2][0]}`) ); expect(mockExecute).toBeCalledWith( - `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${RESOURCE_VERSION}.json`, + `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${RESOURCE_VERSION}.json`, getExpectedOutputPath(testOptions.output, testEntries[2][0], API_VERSION, RESOURCE_VERSION), expectedAtlasBuildOptions, getExpectedVersionOptions(`${SITE_URL}/${testEntries[2][0]}`) @@ -254,21 +254,21 @@ describe('pageBuilder', () => { await buildOpenAPIPages(testEntries, testOptions); expect(mockExecute).toBeCalledWith( - `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${OLDEST_RESOURCE_VERSION}.json`, + `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${OLDEST_RESOURCE_VERSION}.json`, getExpectedOutputPath(testOptions.output, testEntries[0][0], API_VERSION, OLDEST_RESOURCE_VERSION), expectedAtlasBuildOptions, getExpectedVersionOptions(`${SITE_URL}/${testEntries[0][0]}`, OLDEST_RESOURCE_VERSION) ); expect(mockExecute).toBeCalledWith( - `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${LATEST_RESOURCE_VERSION}.json`, + `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${LATEST_RESOURCE_VERSION}.json`, getExpectedOutputPath(testOptions.output, testEntries[0][0], API_VERSION, LATEST_RESOURCE_VERSION), expectedAtlasBuildOptions, getExpectedVersionOptions(`${SITE_URL}/${testEntries[0][0]}`, LATEST_RESOURCE_VERSION) ); expect(mockExecute).toBeCalledWith( - `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${LATEST_RESOURCE_VERSION}.json`, + `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${MOCKED_GIT_HASH}-v2-${LATEST_RESOURCE_VERSION}.json`, getExpectedOutputPath(testOptions.output, testEntries[0][0], API_VERSION, LATEST_RESOURCE_VERSION), expectedAtlasBuildOptions, getExpectedVersionOptions(`${SITE_URL}/${testEntries[0][0]}`, LATEST_RESOURCE_VERSION) @@ -283,7 +283,7 @@ describe('pageBuilder', () => { await buildOpenAPIPages(testEntries, testOptions); expect(mockExecute).toBeCalledWith( - `https://mongodb-mms-prod-build-server.s3.amazonaws.com/openapi/${LAST_SAVED_GIT_HASH}.json`, + `https://mongodb-mms-build-server.s3.amazonaws.com/openapi/${LAST_SAVED_GIT_HASH}.json`, getExpectedOutputPath(testOptions.output, testEntries[0][0]), expectedAtlasBuildOptions, undefined diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts index 5393e2c06..3ec748278 100644 --- a/src/commands/src/helpers/dependency-helpers.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -36,6 +36,7 @@ async function createEnvProdFile( GATSBY_MANIFEST_PATH=${prodDirName}/bundle.zip GATSBY_PARSER_USER=${process.env.USER} GATSBY_BASE_URL=${baseUrl} + GATSBY_MARIAN_URL=${process.env.GATSBY_MARIAN_URL} PATH_PREFIX=${prefix}`, 'utf8' ); diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 1e820094b..9b6cb5876 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -427,6 +427,7 @@ export abstract class JobHandler { GATSBY_TEST_SEARCH_UI: this._config.get('featureFlagSearchUI'), GATSBY_SHOW_CHATBOT: this._config.get('gatsbyUseChatbot'), GATSBY_HIDE_UNIFIED_FOOTER_LOCALE: this._config.get('gatsbyHideUnifiedFooterLocale'), + GATSBY_MARIAN_URL: this._config.get('gatsbyMarianURL'), }; for (const [envName, envValue] of Object.entries(snootyFrontEndVars)) { diff --git a/tests/data/data.ts b/tests/data/data.ts index ec822e2bf..b9aa9251f 100644 --- a/tests/data/data.ts +++ b/tests/data/data.ts @@ -176,7 +176,7 @@ export class TestDataProvider { } static getEnvVarsWithPathPrefixWithFlags(job: Job): string { - return `GATSBY_PARSER_USER=TestUser\nGATSBY_PARSER_BRANCH=${job.payload.branchName}\nPATH_PREFIX=${job.payload.pathPrefix}\nGATSBY_BASE_URL=test\nPREVIEW_BUILD_ENABLED=false\nGATSBY_TEST_SEARCH_UI=false\nGATSBY_SHOW_CHATBOT=false\nGATSBY_HIDE_UNIFIED_FOOTER_LOCALE=true\n`; + return `GATSBY_PARSER_USER=TestUser\nGATSBY_PARSER_BRANCH=${job.payload.branchName}\nPATH_PREFIX=${job.payload.pathPrefix}\nGATSBY_BASE_URL=test\nPREVIEW_BUILD_ENABLED=false\nGATSBY_TEST_SEARCH_UI=false\nGATSBY_SHOW_CHATBOT=false\nGATSBY_HIDE_UNIFIED_FOOTER_LOCALE=true\nGATSBY_MARIAN_URL=test-url\n`; } static getPathPrefixCases(): Array { diff --git a/tests/unit/job/productionJobHandler.test.ts b/tests/unit/job/productionJobHandler.test.ts index b6934df7f..9181b95c9 100644 --- a/tests/unit/job/productionJobHandler.test.ts +++ b/tests/unit/job/productionJobHandler.test.ts @@ -247,7 +247,7 @@ describe('ProductionJobHandler Tests', () => { expect(jobHandlerTestHelper.fileSystemServices.writeToFile).toBeCalledWith( `repos/${jobHandlerTestHelper.job.payload.repoName}/.env.production`, - `GATSBY_PARSER_USER=TestUser\nGATSBY_PARSER_BRANCH=${jobHandlerTestHelper.job.payload.branchName}\nPATH_PREFIX=/\nGATSBY_BASE_URL=test\nPREVIEW_BUILD_ENABLED=false\nGATSBY_TEST_SEARCH_UI=false\nGATSBY_SHOW_CHATBOT=false\nGATSBY_HIDE_UNIFIED_FOOTER_LOCALE=true\n`, + `GATSBY_PARSER_USER=TestUser\nGATSBY_PARSER_BRANCH=${jobHandlerTestHelper.job.payload.branchName}\nPATH_PREFIX=/\nGATSBY_BASE_URL=test\nPREVIEW_BUILD_ENABLED=false\nGATSBY_TEST_SEARCH_UI=false\nGATSBY_SHOW_CHATBOT=false\nGATSBY_HIDE_UNIFIED_FOOTER_LOCALE=true\nGATSBY_MARIAN_URL=test-url\n`, { encoding: 'utf8', flag: 'w' } ); }); diff --git a/tests/utils/jobHandlerTestHelper.ts b/tests/utils/jobHandlerTestHelper.ts index 3a0c223b2..9f39f27e6 100644 --- a/tests/utils/jobHandlerTestHelper.ts +++ b/tests/utils/jobHandlerTestHelper.ts @@ -154,6 +154,7 @@ export class JobHandlerTestHelper { this.config.get.calledWith('featureFlagSearchUI').mockReturnValue('false'); this.config.get.calledWith('gatsbyUseChatbot').mockReturnValue('false'); this.config.get.calledWith('gatsbyHideUnifiedFooterLocale').mockReturnValue('true'); + this.config.get.calledWith('gatsbyMarianURL').mockReturnValue('test-url'); this.repoConnector.checkCommits .calledWith(this.job) .mockReturnValue(TestDataProvider.getCommitCheckValidResponse(this.job));